Elixir - MembraneWebRTC / React: MediaStreams are non-null on one client & null on another & Neither appear

Viewed 124

I'm extending the client used in the videoroom tutorial, but I can't get the remote streams to appear on the client -- either the stream is null or the streams are non-null & nothing shows up.

const useGala = roomId => {
    const { socket } = useContext(SocketContext);
    const permChannel = socket.channel(`room:${roomId}`);

    const [gala, setGala] = useState({
        channel: permChannel,
        localStream: null, 
        peers: [],
        streams: [],
        webrtcEngine: null,
        initialized: false,
    });

    const joinWebrtc = async (engine) => {
        engine.join({ displayName: "Will" });
    }

    const createLocalStream = async (engine) => {
        const localStream = await navigator.mediaDevices.getUserMedia(MEDIA_CONSTRAINTS);
        localStream.getTracks().forEach((track) => {
            engine.addTrack(track, localStream)
        });
        setGala(prev => ({ ...prev, localStream: localStream }));
    }

    const makeEngine = async () => {
        // Create the WebRTC engine
        const engine = new MembraneWebRTC({
            callbacks: {
                onSendMediaEvent: (mediaEvent) => {
                    console.log('gala:onSendMediaEvent: fired');
                    gala.channel.push("mediaEvent", { data: mediaEvent });
                },
                onConnectionError: () => {
                    console.log('on connection error triggered')
                },
                onJoinSuccess: (peerId, peersInRoom) => {
                    console.log("gala:onJoinSuccess: fired");
                    setGala(prev => ({ ...prev, peers: peersInRoom }));
                    createLocalStream(engine);
                },
                onJoinError: (metadata) => {
                    throw `Peer denied.`;
                },
                onTrackReady: ({ stream, peer, metadata }) => {
                    console.log("gala:onTrackReady: fired");
                    const inboundStream = { peerId: `${peer.id}`, stream: stream };
                    // This fires, but the `stream` object is either 
                    // 1. null if on the already joined client
                    // 2. non-null if on the newly joined client
                    setGala(prev => ({
                        ...prev,
                        streams: [...prev.streams, inboundStream]
                    }));
                },
                onTrackAdded: (ctx) => {
                    console.log("hook:onTrackAdded: fired");
                    console.log(ctx);
                },
                onTrackRemoved: (ctx) => { },
                onPeerJoined: (peer) => {
                    console.log("hook:onPeerJoined: fired");

                    setGala(prev => {
                        return {
                            ...prev,
                            peers: [...prev.peers, peer]
                        }
                    });
                },
                onPeerLeft: (peer) => {
                    console.log("hook:onPeerLeft: fired");

                    setGala(prev => {
                        const remainingPeers = prev.peers.filter((p) => p.id !== peer.id);
                        return {
                            ...prev,
                            peers: remainingPeers
                        }
                    });
                },
                onPeerUpdated: (ctx) => { },
            }
        });

        return engine;
    }

    const otherStuff = async (engine) => {
        if (gala.channel.state === 'joined' && !!engine) {
            // Setup a tracker for inbound media
            gala.channel.on("mediaEvent", (event) => {
                // For already-joined client, does not fire if new peer joins
                // For new peer, fires
                engine.receiveMediaEvent(event.data)
            });

            setGala(prev => ({
                ...prev,
                initialized: true,
                webrtcEngine: engine
            }))
        }
    }

    const doEverything = async () => {
        let engine;
        if (!gala.webrtcEngine) {
            engine = await makeEngine();
        }

        // Join the server with the new stream
        const phoenixChannelPushResult = async (push) => {
            return new Promise((resolve, reject) => {
                push
                    .receive("ok", (response) => resolve(response))
                    .receive("error", (response) => reject(response));
            });
        };

        if (gala.channel.state !== 'joining' && gala.channel.state !== 'joined' && (!!engine || gala.webrtcEngine)) {
            await phoenixChannelPushResult(gala.channel.join());
            if (engine) {
                await joinWebrtc(engine);
                return engine
            } else {
                return gala.webrtcEngine;
            }
        }

    }

    const all = useCallback(async () => {
        doEverything()
            .then(engine => otherStuff(engine));
    }, [gala.channel]);

    const doStuffWithChannel = useCallback(async () => {
        if (gala?.streams?.length > 0) {
            // Setup a tracker for inbound media
            // Set this up to see if it would fire, but nothing
            gala.channel.on("mediaEvent", (event) => {
                gala.webrtcEngine.receiveMediaEvent(event.data)
            });
        }
    }, [gala.streams, gala.channel]);

    useEffect(() => {
        if (!socket) return;
        all();
        doStuffWithChannel();
    }, [socket]);

    return gala;
}

I'm taking all the gala.streams entries and loading them into a MediaElement. Even if the streams are non-null, nothing appears:

const MediaElement = ({ peerId, gala }) => {
    const audioRef = useRef();
    const videoRef = useRef();

    useEffect(() => {
        if (gala?.streams?.length > 0) {
            audioRef.current.srcObject = gala?.streams[gala?.streams?.length - 1]?.stream;
            videoRef.current.srcObject = gala?.streams[gala?.streams?.length - 1]?.stream;
        }
    }, [gala?.streams, gala.channel, gala.webrtcEngine]);

    return (
        <div className='col-span-full col-start-3 row-span-3' key={peerId}>
            <p className='text-white'>{peerId}</p>
            <audio id={`audio-${peerId}`} ref={audioRef} volume="true" autoPlay></audio>
            <video id={`video-${peerId}`} ref={videoRef} playsInline autoPlay></video>
        </div>
    );
}

export default MediaElement;

I believe the error has something to do with how I'm managing the state of the webRtcEngine, because this is the key differentiator from the vanilla JavaScript example.

I have no idea how to fix this though. How can I get the MediaStreams to appear?

1 Answers

I would write this as a comment, but I don't have enough reputation.

At first glance, it seems like your code is all right. It might be an issue with React (some sort of race condition) rather than membrane-webrtc.

One question, though: why are you only loading the last stream?

    useEffect(() => {
            if (gala?.streams?.length > 0) {
                audioRef.current.srcObject = gala?.streams[gala?.streams?.length - 1]?.stream;
                videoRef.current.srcObject = gala?.streams[gala?.streams?.length - 1]?.stream;
            }
        }, [gala?.streams, gala.channel, gala.webrtcEngine]);

Related