Send MediaRecorder audio chunks with Websockets

I’m trying to send audio chunks recorded with MediaRecorder to the server and then to another client using Websockets and play it there.

The first chunk gets played normally but then I get Error decoding audio data: EncodingError: Failed to execute 'decodeAudioData' on 'BaseAudioContext': Unable to decode audio data

This is my code:

Recording:

      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
      const mediaRecorder = new MediaRecorder(stream, {
        mimeType: "audio/webm",
      });
      mediaRecorder.ondataavailable = async (event) => {
        if (event.data.size > 0) {
          const b64 = await blobToBase64(event.data);
          sendMessage({
            payload: {
              base64: b64,
            },
          });
        }
      };

      mediaRecorder.start(250);

Playing:

   const arrayBuffer = base64ToArrayBuffer(message.payload.base64);
   const audioContext = new AudioContext();
   const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
   const source = audioContext.createBufferSource();
   source.buffer = audioBuffer;
   source.connect(audioContext.destination);
   source.start();

Helper functions:

const base64ToArrayBuffer = (base64: string): ArrayBuffer => {
  const binaryString = window.atob(base64);
  const len = binaryString.length;
  const bytes = new Uint8Array(len);
  for (let i = 0; i < len; i++) {
    bytes[i] = binaryString.charCodeAt(i);
  }
  return bytes.buffer;
};

  const blobToBase64 = (blob: Blob): Promise<string> => {
    return new Promise((resolve, reject) => {
      const reader = new FileReader();
      reader.readAsDataURL(blob);
      reader.onloadend = () => resolve(reader.result as string);
      reader.onerror = reject;
    });
  };