How do you initialize a WebRTC call using the transceiver API but only enable audio&video later after signaling completed?

Viewed 172

I am trying to first connect two WebRTC peers. Once the connection is established I want to give the users on both sides the option to enable/disable video and audio. This should happen without triggering the signaling process again.

I do run into an issue though: If I call replaceTrack(audioTack) the remote peer will not playback audio until I also call replaceTrack(video).

I am unsure why this happen and can not find any clue in the documentation. It does play fine after 10 seconds once I also attach the video track. Without video track there is no audio playback. Why?

function createVideoElement() {
  const vid = document.createElement("video")
  vid.width = 320;
  vid.controls = true;
  vid.autoplay = true;
  const root = document.body;
  document.body.appendChild(vid);
  return vid;
}

async function RunTestInit() {

  console.log("get media access");
  const p1_stream_out = await navigator.mediaDevices.getUserMedia({
    video: true,
    audio: true
  });
  const p2_stream_out = await navigator.mediaDevices.getUserMedia({
    video: true,
    audio: true
  });


  console.log("stream setup");
  const p1_stream_in = new MediaStream();
  const p2_stream_in = new MediaStream();

  const p1_video_in = createVideoElement();
  const p2_video_in = createVideoElement();

  console.log("peer setup");
  const p1 = new RTCPeerConnection();
  const p2 = new RTCPeerConnection();
  const p1_tca = p1.addTransceiver("audio", {
    direction: "sendrecv"
  });
  const p1_tcv = p1.addTransceiver("video", {
    direction: "sendrecv"
  });


  p1.onicecandidate = (ev) => {
    p2.addIceCandidate(ev.candidate);
  }
  p2.onicecandidate = (ev) => {
    p1.addIceCandidate(ev.candidate);
  }

  p1.onconnectionstatechange = (ev) => {
    console.log("p1 state: ", p1.connectionState);
  }
  p2.onconnectionstatechange = async (ev) => {
    console.log("p2 state: ", p2.connectionState);
  }

  p1.onnegotiationneeded = () => {
    //triggers once
    console.warn("p1.onnegotiationneeded");
  }

  p2.onnegotiationneeded = () => {
    //should never trigger
    console.warn("p2.onnegotiationneeded");
  }

  p1.ontrack = (ev) => {
    console.log("p1.ontrack", ev);
    p1_stream_in.addTrack(ev.track);
    p1_video_in.srcObject = p1_stream_in;
  }
  p2.ontrack = (ev) => {
    console.log("p2.ontrack", ev);
    p2_stream_in.addTrack(ev.track);
    p2_video_in.srcObject = p2_stream_in;
  }
  console.log("signaling");
  const offer = await p1.createOffer();
  await p1.setLocalDescription(offer);
  await p2.setRemoteDescription(offer);
  const p2_tca = p2.getTransceivers()[0];
  const p2_tcv = p2.getTransceivers()[1];

  p2_tca.direction = "sendrecv"
  p2_tcv.direction = "sendrecv"

  const answer = await p2.createAnswer();
  await p2.setLocalDescription(answer);
  await p1.setRemoteDescription(answer);
  console.log("signaling done");

  //send audio from p2 to p1 (direction doesn't matter)
  //after this runs nothing will happen and no audio plays
  setTimeout(async () => {
    await p2_tca.sender.replaceTrack(p2_stream_out.getAudioTracks()[0]);
    console.warn("audio playback should start now but nothing happens");
  }, 1000);

  //audio starts playing once this runs
  setTimeout(async () => {
    //uncomment this and it works just fine
    await p2_tcv.sender.replaceTrack(p2_stream_out.getVideoTracks()[0]);
    console.warn("now audio playback starts");
  }, 10000);
}

function start() {
  setTimeout(async () => {
    console.log("Init test case");
    await RunTestInit();
  }, 1);
}

Same example in the js fiddle (needs camera and microphone access): https://jsfiddle.net/vnztcx5p/5/

Once audio works this will cause an echo.

3 Answers

You can enable/disable audio and video tracks, so you dont have to renegotiate. Note that this tracks have to be added before negotiation starts. You can achieve it with:

mediaStream.getAudioTracks()[0].enabled = false; // or true to enable it.

Or if you want to disable video:

mediaStream.getVideoTracks()[0].enabled = false; // or true to enable it.

Here is the documentation

getAudioTracks()

getVideoTracks()

I got this working. It looks like more a problem with how HTMLVideoElement works rather than WebRTC.

If I set

p1_video_in.srcObject = p1_stream_in;
p2_video_in.srcObject = p2_stream_in;

before I add the tracks to the stream it works.

Complete example looks like this:

function createVideoElement() {
  const vid = document.createElement("video")
  vid.width = 320;
  vid.controls = true;
  vid.autoplay = true;
  const root = document.body;
  document.body.appendChild(vid);
  return vid;
}

async function RunTestInit() {

  console.log("get media access");
  const p1_stream_out = await navigator.mediaDevices.getUserMedia({
    video: true,
    audio: true
  });
  const p2_stream_out = await navigator.mediaDevices.getUserMedia({
    video: true,
    audio: true
  });


  console.log("stream setup");
  const p1_stream_in = new MediaStream();
  const p2_stream_in = new MediaStream();

  const p1_video_in = createVideoElement();
  const p2_video_in = createVideoElement();
  p1_video_in.srcObject = p1_stream_in;
  p2_video_in.srcObject = p2_stream_in;

  console.log("peer setup");
  const p1 = new RTCPeerConnection();
  const p2 = new RTCPeerConnection();
  const p1_tca = p1.addTransceiver("audio", {
    direction: "sendrecv"
  });
  const p1_tcv = p1.addTransceiver("video", {
    direction: "sendrecv"
  });


  p1.onicecandidate = (ev) => {
    p2.addIceCandidate(ev.candidate);
  }
  p2.onicecandidate = (ev) => {
    p1.addIceCandidate(ev.candidate);
  }

  p1.onconnectionstatechange = (ev) => {
    console.log("p1 state: ", p1.connectionState);
  }
  p2.onconnectionstatechange = async (ev) => {
    console.log("p2 state: ", p2.connectionState);
  }

  p1.onnegotiationneeded = () => {
    //triggers once
    console.warn("p1.onnegotiationneeded");
  }

  p2.onnegotiationneeded = () => {
    //should never trigger
    console.warn("p2.onnegotiationneeded");
  }

  p1.ontrack = (ev) => {
    console.log("p1.ontrack", ev);
    p1_stream_in.addTrack(ev.track);
  }
  p2.ontrack = (ev) => {
    console.log("p2.ontrack", ev);
    p2_stream_in.addTrack(ev.track);
  }
  console.log("signaling");
  const offer = await p1.createOffer();
  await p1.setLocalDescription(offer);
  await p2.setRemoteDescription(offer);
  const p2_tca = p2.getTransceivers()[0];
  const p2_tcv = p2.getTransceivers()[1];

  p2_tca.direction = "sendrecv"
  p2_tcv.direction = "sendrecv"

  const answer = await p2.createAnswer();
  await p2.setLocalDescription(answer);
  await p1.setRemoteDescription(answer);
  console.log("signaling done");

  //send audio from p2 to p1 (direction doesn't matter)
  //after this runs nothing will happen and no audio plays
  setTimeout(async () => {
    await p2_tca.sender.replaceTrack(p2_stream_out.getAudioTracks()[0]);
    console.warn("audio playback should start now but nothing happens");
  }, 1000);

  //audio starts playing once this runs
  setTimeout(async () => {
    //uncomment this and it works just fine
    await p2_tcv.sender.replaceTrack(p2_stream_out.getVideoTracks()[0]);
    console.warn("now audio playback starts");
  }, 10000);
}

function start() {
  setTimeout(async () => {
    console.log("Init test case");
    await RunTestInit();
  }, 1);
}
Related