how to add audio to video streaming using media source extension

Viewed 652

I have a mp4 file sample.mp4. I used mp4box to convert it into segments and mpd using how to create a mpd file using MP4Box Now my code is from this source. After Creating the segments the video is played in the browser. But there is no audio.

The files formed are as follows : Files formed after conversion of sample.mp4 to mpd using mp4box

I have two queries here.

Firstly, How can I add audio to this video using javascript in MSE (Media Source Extension)? The video is playing on a mute.

Secondly, The new format files are of the name sample_dash_track1_init.mp4 and sample_dash_track2_init.mp4. While the video plays with the first file and its segments, what is the use of second file and its segments?

 `<!DOCTYPE html>
 <html lang="en">
 <head>
 <meta charset="UTF-8">
  <title>MSE Demo</title>
 </head>
 <body>
 <h1>MSE Demo</h1>
  <div>
   <video controls width="80%" ></video>
  </div>

  <script type="text/javascript">
  (function() {
  var baseUrl = 'http://beta.insidesoftwares.com/development/video/sam/';
  var initUrl = baseUrl + 'sample_dash_track1_init.mp4';
  var templateUrl = baseUrl + 'sample_dash_track1_$Number$.m4s';
  var sourceBuffer;
  var index = 1;
  var numberOfChunks = 13;
  var video = document.querySelector('video');

  if (!window.MediaSource) {
    console.error('No Media Source API available');
    return;
  }

  var ms = new MediaSource();
  video.src = window.URL.createObjectURL(ms);
  ms.addEventListener('sourceopen', onMediaSourceOpen);

  function onMediaSourceOpen() {
    sourceBuffer = ms.addSourceBuffer('video/mp4; codecs="avc1.4d401f"');
    sourceBuffer.addEventListener('updateend', nextSegment);

    GET(initUrl, appendToBuffer);

    video.play();
  }

  function nextSegment() {
    var url = templateUrl.replace('$Number$', index);
    GET(url, appendToBuffer);
    index++;
    if (index > numberOfChunks) {
      sourceBuffer.removeEventListener('updateend', nextSegment);
    }
  }

  function appendToBuffer(videoChunk) {
    if (videoChunk) {
      sourceBuffer.appendBuffer(new Uint8Array(videoChunk));
    }
  }

  function GET(url, callback) {
    var xhr = new XMLHttpRequest();
    xhr.open('GET', url);
    xhr.responseType = 'arraybuffer';

    xhr.onload = function(e) {
      if (xhr.status != 200) {
        console.warn('Unexpected status code ' + xhr.status + ' for ' + url);
        return false;
      }
      callback(xhr.response);
    };

    xhr.send();
   }
   })();
  </script>
  </body>
  </html>`
1 Answers

I am guessing that sample_dash_track1* contains the video and sample_dash_track2* contains the audio. So your hand written JavaScript player plays only video because there is only video in sample_dash_track1*.

Although I like your JavaScript player - you may want to take a look at https://github.com/Dash-Industry-Forum/dash.js?

Otherwise you would have to parse the mpd file and dynamically remux which is a significant amount of work that has been done before for example in the DASH.js library.

Related