Parsing and decoding socket stream to create HTML video&audio – HLS

I have a socket stream constantly sending me partial Binary data as image below,

When I am console.log() this data it appears as something as base64 as sample below,

AAAAZG1vb2YAAAAQbWZoZAAAAAAAfPOgAAAATHRyYWYAAAAcdGZoZAAAADgAAAACAAAAFQAAAPICAAAAAAAAFHRmZHQBAAAAAAABk8M0zYoAAAAUdHJ1bgAAAAEAAAABAAAAbAAAAPptZGF0ISmEVZ6ND2MRAANN1GiIIklXQDem76zqqo2EiHkWiR+uXJysLrZje6a1GXMBmUDur7+YXmqXN+YS8gJoa4AbTGj7+mhrEF6/ukUgOSiw8WVJO13cmCkXn5T1K6Qqycqf44mCxIyZ5JS5XPtcgkgC8NOaFAl8iZMMWOC8wkqCjgYISASB4o4YXRiX3gnZriKVLNARKgjppNYYWXUeqkr+F3douFTAb5EQmOeaCjkEGRpIN76X3cM8rqVrBoLNRJS8IdgNTtQ4ZOF+bJLVEDJAToO+/spvj9q4LIqEnLAegZzxJtvbgxghs9Kz/n4AAAAAAocAAABkbW9vZgAAABBtZmhkAAAAAAB886IAAABMdHJhZgAAABx0ZmhkAAAAOAAAAAEAAAAoAAAEUgEBAAAAAAAUdGZkdAEAAAAAAAGTwzTNoAAAABR0cnVuAAAAAQAAAAEAAABsAAAEWm1kYXQAAABIQZtmCMf/Im1rWeR69T8zkFAmfAfoK8NKFi88JYLf32iLNVjzErZ/jjpHs3mHQ+sQ/ipPrzUWwp1LnhIG2lxoYcZ7SxPjTeSAAAABEUEBsm2YIQf/wsIlmx/ckvbQeYSAoxgV0yy9OJub+vzs7P8rfaX8r/qwRhUptexGqLAWLZ8qVu+n8lEMDTGjPWAN2bmqf/Bmupg0+40wkbI4jS1cvbvQO8L6CnptvWcuBGuCOy8OBwAWiRY7AcSbqB0J3BV+6iQqH+mhdrcSOSTqq/Rt6Kag5IDLYj5RlIxRvvk3Orq/4dOQI/Z1g8pB+FwjFQJEXJcHK7Miz1lUyeuUiqP6r+tdS0Eo9pWqEWSL8FnLVbyY+lUlyVjmzs8gckg34iJUX2bcTfYLEGQrwyuliu7rdK1i8n/xCLBak2WdZUcIvivseyNvKKV2VoXgjncOmqVLivt7K9rDauQ+HelV0AAAAN5BAPObZgt/A9P18qInrt74MHpdYVkaby8WLU2oUMpOye8TVo2vAeKKt8QTr0dTWGkwqlmK0K6OrXHaFrdhZFDbsCDBoq18ZSnfpdntVKRsqotnhrRmcVavB33t6jJM9g/WXSukYseMP3oPBGEBlTOxHPD7ahHOrBE3XOe8Ib21ZWYc0Ysu4zpjJfQMoU4BSYbpVdEmJk8WKuJN9q/P3rhi/VB9AucFHt54Ib8cfS1GuopjDJGP5T0RmwvtWUBP3tEZwG0IGZ7nt5BC8LTsPqPzZD/MgybWAdeu65lOS3AAAAD3QQBX5tmCV/8BAjuEzqGAgRTDCDEJkhNuKUWg4YqWV3cDLk8wzIABpBP91qOptVP9MxsSHfSp2+Zo9PG9f7mAjSLRK39s8H9wDGKnDa8rbCaWzMaCpWbpAota7e+AuBmgwR0fYs0E9E3yrpX7F6vAm5+8+NarmqxCwsDjukTFC8md41BMz1reOYWPizpp7aRVA8W2QsJXdnaxTxmptg+IMm6Tw1nqu3nIVTHOQjfvRsmABHaEqPBvmR/FuyyaKPLj/XWYtAhr129J1Qm9x02xs1hHLI6kalUooR1lssVaFBRT6I/Qf7MOfJi8u9D/zTbRdKWC7RBqCAAAAMBBAHmm2YJX/wBwYiJBlNATOV1c5lGm8ZdHs0GFScWsHt9sgK/lIaAV9P7/hN/lN4OOZiPWVWUoF5dWtoesWohef3gj/dWZHd9Ilp2wRemXqzZlwzNnYypV+74zw5uIRbi8TgfPLtK2kslgjQQno22YV8mLQUeLJv1iTSINAmMz6t6L1Uh0WVgy+qncooflaGq2U1eIAK2vttOPlha2uuQrqKkBsOl8RJ4flLS6tEUqzEC2ePJfHmIAP4vp8TxSxjsAAAAvQQAlKbZglf8AKXFDs2tqEIh0VAKRnoKFzIy+i66Mc/tlCT9Om9glbz5BUacpJegAAAAZQQAtmbZglf8AAjVa7iVvO/r/qGsYP7ZfBgAAACBlZ3djAAABk8M0zaAABikyhkOHIAAAAZPDNM5S

When I parse this data with atob() it returns a chunk string always starts with dmoof mfhd prefix which I believe it’s something about MP4 encoding.

What I also know about this data has vc: h264 ac: aac encoding and avc1.42E01E, mp4a.40.2 codecs in order for me to decode it.

I’ve tried multiple methods including hls.js, jmuxer (https://github.com/samirkumardas/jmuxer) MediaSource() but couldn’t be successful to create video&audio in my HTML page.

This code below is the most closest try I got,

<video id="video" controls autoplay></video>
<script>
    // Convert base64 string to ArrayBuffer
    function base64ToArrayBuffer(base64) {
      const binaryString = atob(base64);
      const bytes = new Uint8Array(binaryString.length);
      for (let i = 0; i < binaryString.length; i++) {
        bytes[i] = binaryString.charCodeAt(i);
      }
      return bytes.buffer;
    }

    class VideoStreamHandler {
      constructor() {
        this.mediaSource = null;
        this.sourceBuffer = null;
        this.videoElement = document.getElementById('video');
        this.queue = [];
        this.isInitialized = false;

        // Bind methods to maintain correct context
        this.onSourceOpen = this.onSourceOpen.bind(this);
        this.processQueue = this.processQueue.bind(this);
      }

      init() {
        this.mediaSource = new MediaSource();
        this.videoElement.src = URL.createObjectURL(this.mediaSource);
        this.mediaSource.addEventListener('sourceopen', this.onSourceOpen, false);
      }

      onSourceOpen() {
        try {
          // Use a more generic codec string
          this.sourceBuffer = this.mediaSource.addSourceBuffer('video/mp4; codecs="avc1.42E01E, mp4a.40.2"');
          
          // Add event listeners to handle buffer updates
          this.sourceBuffer.addEventListener('updateend', this.processQueue);
          
          this.isInitialized = true;
          this.processQueue();
        } catch (e) {
          console.error('Error creating SourceBuffer:', e);
        }
      }

      processQueue() {
        if (this.queue.length > 0 && !this.sourceBuffer.updating) {
          const arrayBuffer = this.queue.shift();
          try {
            this.sourceBuffer.appendBuffer(arrayBuffer);
          } catch (error) {
            console.error('Error appending buffer:', error);
          }
        }
      }

      handleVideoStream(base64Data) {
        const arrayBuffer = base64ToArrayBuffer(base64Data);

        if (!this.isInitialized) {
          this.init();
        }

        // Always queue the buffer
        this.queue.push(arrayBuffer);

        // Try to process the queue
        if (!this.sourceBuffer.updating) {
          this.processQueue();
        }
      }
    }

    // Create a global instance of the handler
    const videoStreamHandler = new VideoStreamHandler();

    // Listen for video stream messages
    window.addEventListener('message', (msg) => {
      if (msg.data.action === 'VIDEO_STREAM') {
        var b64data = msg.data.data;
        videoStreamHandler.handleVideoStream(b64data);
      }
    });
  </script>

It updates the src attribute of the video tag with a blob url as,

<video id="video" controls="" autoplay="" src="blob:http://localhost/603a5c88-9aed-4f18-800f-885805fc1d5b"></video>

But it returns the error of

Error appending buffer: InvalidStateError: Failed to execute 'appendBuffer' on 'SourceBuffer': This SourceBuffer has been removed from the parent media source.

for the part of the code,

this.sourceBuffer.appendBuffer(arrayBuffer);

as image below,

I am not really experienced of http live streaming side of web development, can’t wait to hear your suggestions to try to achieve creating video&audio from the data. Thanks.