Port some changes from group calls branch to develop (#2001)

* Add some useful getters Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Add removeLocalFeed() Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Don't updateStream() if they're the same Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Add isSpeaking() Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Improve speaking detection using history Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Make code for placing and answering calls more flexible Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me> * Correctly log stream id Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> * Remove mistaken parameter Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> * Add a unit Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com> Co-authored-by: Robert Long <robert@robertlong.me>
2025-11-26 17:03:12 +03:00 · 2021-10-26 20:59:15 +02:00
parent 51c776f51e
commit c35cb57a79
2 changed files with 190 additions and 70 deletions
--- a/src/webrtc/call.ts
+++ b/src/webrtc/call.ts
@@ -416,10 +416,26 @@ export class MatrixCall extends EventEmitter {
        return this.localUsermediaFeed?.stream;
    }

-    private get localScreensharingStream(): MediaStream {
+    public get localScreensharingStream(): MediaStream {
        return this.localScreensharingFeed?.stream;
    }

+    public get remoteUsermediaFeed(): CallFeed {
+        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
+    }
+
+    public get remoteScreensharingFeed(): CallFeed {
+        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
+    }
+
+    public get remoteUsermediaStream(): MediaStream {
+        return this.remoteUsermediaFeed?.stream;
+    }
+
+    public get remoteScreensharingStream(): MediaStream {
+        return this.remoteScreensharingFeed?.stream;
+    }
+
    private getFeedByStreamId(streamId: string): CallFeed {
        return this.getFeeds().find((feed) => feed.stream.id === streamId);
    }
@@ -551,15 +567,20 @@ export class MatrixCall extends EventEmitter {
        logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`);
    }

-    private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
+    private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
        const userId = this.client.getUserId();

+        // TODO: Find out what is going on here
+        // why do we enable audio (and only audio) tracks here? -- matthew
+        setTracksEnabled(stream.getAudioTracks(), true);
+
        // We try to replace an existing feed if there already is one with the same purpose
        const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose);
        if (existingFeed) {
            existingFeed.setNewStream(stream);
        } else {
-            this.feeds.push(new CallFeed({
+            this.pushLocalFeed(
+                new CallFeed({
                    client: this.client,
                    roomId: this.roomId,
                    audioMuted: stream.getAudioTracks().length === 0,
@@ -567,35 +588,66 @@ export class MatrixCall extends EventEmitter {
                    userId,
                    stream,
                    purpose,
-            }));
+                }),
+                addToPeerConnection,
+            );
            this.emit(CallEvent.FeedsChanged, this.feeds);
        }
+    }

-        // TODO: Find out what is going on here
-        // why do we enable audio (and only audio) tracks here? -- matthew
-        setTracksEnabled(stream.getAudioTracks(), true);
+    /**
+     * Pushes supplied feed to the call
+     * @param {CallFeed} callFeed to push
+     * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection
+     */
+    public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void {
+        this.feeds.push(callFeed);

        if (addToPeerConnection) {
-            const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ?
+            const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ?
                this.usermediaSenders : this.screensharingSenders;
            // Empty the array
            senderArray.splice(0, senderArray.length);

-            this.emit(CallEvent.FeedsChanged, this.feeds);
-            for (const track of stream.getTracks()) {
+            for (const track of callFeed.stream.getTracks()) {
                logger.info(
                    `Adding track (` +
                    `id="${track.id}", ` +
                    `kind="${track.kind}", ` +
-                    `streamId="${stream.id}", ` +
-                    `streamPurpose="${purpose}"` +
+                    `streamId="${callFeed.stream.id}", ` +
+                    `streamPurpose="${callFeed.purpose}"` +
                    `) to peer connection`,
                );
-                senderArray.push(this.peerConn.addTrack(track, stream));
+                senderArray.push(this.peerConn.addTrack(track, callFeed.stream));
            }
        }

-        logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`);
+        logger.info(
+            `Pushed local stream ` +
+            `(id="${callFeed.stream.id}", ` +
+            `active="${callFeed.stream.active}", ` +
+            `purpose="${callFeed.purpose}")`,
+        );
+
+        this.emit(CallEvent.FeedsChanged, this.feeds);
+    }
+
+    /**
+     * Removes local call feed from the call and its tracks from the peer
+     * connection
+     * @param callFeed to remove
+     */
+    public removeLocalFeed(callFeed: CallFeed): void {
+        const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia
+            ? this.usermediaSenders
+            : this.screensharingSenders;
+
+        for (const sender of senderArray) {
+            this.peerConn.removeTrack(sender);
+        }
+        // Empty the array
+        senderArray.splice(0, senderArray.length);
+        this.deleteFeedByStream(callFeed.stream);
    }

    private deleteAllFeeds(): void {
@@ -760,11 +812,27 @@ export class MatrixCall extends EventEmitter {
            this.waitForLocalAVStream = true;

            try {
-                const mediaStream = await this.client.getMediaHandler().getUserMediaStream(
+                const stream = await this.client.getMediaHandler().getUserMediaStream(
                    answerWithAudio, answerWithVideo,
                );
                this.waitForLocalAVStream = false;
-                this.gotUserMediaForAnswer(mediaStream);
+                const usermediaFeed = new CallFeed({
+                    client: this.client,
+                    roomId: this.roomId,
+                    userId: this.client.getUserId(),
+                    stream,
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                    audioMuted: stream.getAudioTracks().length === 0,
+                    videoMuted: stream.getVideoTracks().length === 0,
+                });
+
+                const feeds = [usermediaFeed];
+
+                if (this.localScreensharingFeed) {
+                    feeds.push(this.localScreensharingFeed);
+                }
+
+                this.answerWithCallFeeds(feeds);
            } catch (e) {
                if (answerWithVideo) {
                    // Try to answer without video
@@ -782,6 +850,14 @@ export class MatrixCall extends EventEmitter {
        }
    }

+    public answerWithCallFeeds(callFeeds: CallFeed[]): void {
+        if (this.inviteOrAnswerSent) return;
+
+        logger.debug(`Answering call ${this.callId}`);
+
+        this.gotCallFeedsForAnswer(callFeeds);
+    }
+
    /**
     * Replace this call with a new call, e.g. for glare resolution. Used by
     * MatrixClient.
@@ -793,7 +869,7 @@ export class MatrixCall extends EventEmitter {
            newCall.waitForLocalAVStream = true;
        } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) {
            logger.debug("Handing local stream to new call");
-            newCall.gotUserMediaForAnswer(this.localUsermediaStream);
+            newCall.gotCallFeedsForAnswer(this.getLocalFeeds());
        }
        this.successor = newCall;
        this.emit(CallEvent.Replaced, newCall);
@@ -865,7 +941,7 @@ export class MatrixCall extends EventEmitter {
                if (this.hasLocalUserMediaAudioTrack) return;
                if (this.hasLocalUserMediaVideoTrack) return;

-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
            } else if (upgradeAudio) {
                if (this.hasLocalUserMediaAudioTrack) return;

@@ -931,7 +1007,7 @@ export class MatrixCall extends EventEmitter {
            try {
                const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                if (!stream) return false;
-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
                return true;
            } catch (err) {
                this.emit(CallEvent.Error,
@@ -973,7 +1049,7 @@ export class MatrixCall extends EventEmitter {
                });
                sender.replaceTrack(track);

-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);

                return true;
            } catch (err) {
@@ -1133,13 +1209,9 @@ export class MatrixCall extends EventEmitter {
        setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted);
    }

-    /**
-     * Internal
-     * @param {Object} stream
-     */
-    private gotUserMediaForInvite = async (stream: MediaStream): Promise<void> => {
+    private gotCallFeedsForInvite(callFeeds: CallFeed[]): void {
        if (this.successor) {
-            this.successor.gotUserMediaForAnswer(stream);
+            this.successor.gotCallFeedsForAnswer(callFeeds);
            return;
        }
        if (this.callHasEnded()) {
@@ -1147,12 +1219,15 @@ export class MatrixCall extends EventEmitter {
            return;
        }

-        this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
+        for (const feed of callFeeds) {
+            this.pushLocalFeed(feed);
+        }
+
        this.setState(CallState.CreateOffer);

        logger.debug("gotUserMediaForInvite");
        // Now we wait for the negotiationneeded event
-    };
+    }

    private async sendAnswer(): Promise<void> {
        const answerContent = {
@@ -1201,12 +1276,15 @@ export class MatrixCall extends EventEmitter {
        this.sendCandidateQueue();
    }

-    private gotUserMediaForAnswer = async (stream: MediaStream): Promise<void> => {
-        if (this.callHasEnded()) {
-            return;
+    private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise<void> {
+        if (this.callHasEnded()) return;
+
+        this.waitForLocalAVStream = false;
+
+        for (const feed of callFeeds) {
+            this.pushLocalFeed(feed);
        }

-        this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
        this.setState(CallState.CreateAnswer);

        let myAnswer;
@@ -1234,7 +1312,7 @@ export class MatrixCall extends EventEmitter {
            this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
            return;
        }
-    };
+    }

    /**
     * Internal
@@ -1976,15 +2054,41 @@ export class MatrixCall extends EventEmitter {
     * @throws if have passed audio=false.
     */
    public async placeCall(audio: boolean, video: boolean): Promise<void> {
-        logger.debug(`placeCall audio=${audio} video=${video}`);
        if (!audio) {
            throw new Error("You CANNOT start a call without audio");
        }
+        this.setState(CallState.WaitLocalMedia);
+
+        try {
+            const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
+            const callFeed = new CallFeed({
+                client: this.client,
+                roomId: this.roomId,
+                userId: this.client.getUserId(),
+                stream,
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                audioMuted: stream.getAudioTracks().length === 0,
+                videoMuted: stream.getVideoTracks().length === 0,
+            });
+            await this.placeCallWithCallFeeds([callFeed]);
+        } catch (e) {
+            this.getUserMediaFailed(e);
+            return;
+        }
+    }
+
+    /**
+     * Place a call to this room with call feed.
+     * @param {CallFeed[]} callFeeds to use
+     * @throws if you have not specified a listener for 'error' events.
+     * @throws if have passed audio=false.
+     */
+    public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise<void> {
        this.checkForErrorListener();
+        this.direction = CallDirection.Outbound;
+
        // XXX Find a better way to do this
        this.client.callEventHandler.calls.set(this.callId, this);
-        this.setState(CallState.WaitLocalMedia);
-        this.direction = CallDirection.Outbound;

        // make sure we have valid turn creds. Unless something's gone wrong, it should
        // poll and keep the credentials valid so this should be instant.
@@ -1996,14 +2100,7 @@ export class MatrixCall extends EventEmitter {
        // create the peer connection now so it can be gathering candidates while we get user
        // media (assuming a candidate pool size is configured)
        this.peerConn = this.createPeerConnection();
-
-        try {
-            const mediaStream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
-            this.gotUserMediaForInvite(mediaStream);
-        } catch (e) {
-            this.getUserMediaFailed(e);
-            return;
-        }
+        this.gotCallFeedsForInvite(callFeeds);
    }

    private createPeerConnection(): RTCPeerConnection {
--- a/src/webrtc/callFeed.ts
+++ b/src/webrtc/callFeed.ts
@@ -19,8 +19,9 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes";
 import { MatrixClient } from "../client";
 import { RoomMember } from "../models/room-member";

-const POLLING_INTERVAL = 250; // ms
-const SPEAKING_THRESHOLD = -60; // dB
+const POLLING_INTERVAL = 200; // ms
+export const SPEAKING_THRESHOLD = -60; // dB
+const SPEAKING_SAMPLE_COUNT = 8; // samples

 export interface ICallFeedOpts {
    client: MatrixClient;
@@ -43,6 +44,7 @@ export class CallFeed extends EventEmitter {
    public stream: MediaStream;
    public userId: string;
    public purpose: SDPStreamMetadataPurpose;
+    public speakingVolumeSamples: number[];

    private client: MatrixClient;
    private roomId: string;
@@ -65,6 +67,7 @@ export class CallFeed extends EventEmitter {
        this.purpose = opts.purpose;
        this.audioMuted = opts.audioMuted;
        this.videoMuted = opts.videoMuted;
+        this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);

        this.updateStream(null, opts.stream);

@@ -78,6 +81,8 @@ export class CallFeed extends EventEmitter {
    }

    private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
+        if (newStream === oldStream) return;
+
        if (oldStream) {
            oldStream.removeEventListener("addtrack", this.onAddTrack);
            this.measureVolumeActivity(false);
@@ -152,6 +157,10 @@ export class CallFeed extends EventEmitter {
        return this.stream.getVideoTracks().length === 0 || this.videoMuted;
    }

+    public isSpeaking(): boolean {
+        return this.speaking;
+    }
+
    /**
     * Replaces the current MediaStream with a new one.
     * This method should be only used by MatrixCall.
@@ -167,6 +176,7 @@ export class CallFeed extends EventEmitter {
     */
    public setAudioMuted(muted: boolean): void {
        this.audioMuted = muted;
+        this.speakingVolumeSamples.fill(-Infinity);
        this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
    }

@@ -191,6 +201,7 @@ export class CallFeed extends EventEmitter {
            this.volumeLooper();
        } else {
            this.measuringVolumeActivity = false;
+            this.speakingVolumeSamples.fill(-Infinity);
            this.emit(CallFeedEvent.VolumeChanged, -Infinity);
        }
    }
@@ -199,10 +210,9 @@ export class CallFeed extends EventEmitter {
        this.speakingThreshold = threshold;
    }

-    private volumeLooper(): void {
+    private volumeLooper = () => {
        if (!this.analyser) return;

-        this.volumeLooperTimeout = setTimeout(() => {
        if (!this.measuringVolumeActivity) return;

        this.analyser.getFloatFrequencyData(this.frequencyBinCount);
@@ -214,16 +224,29 @@ export class CallFeed extends EventEmitter {
            }
        }

+        this.speakingVolumeSamples.shift();
+        this.speakingVolumeSamples.push(maxVolume);
+
        this.emit(CallFeedEvent.VolumeChanged, maxVolume);
-            const newSpeaking = maxVolume > this.speakingThreshold;
+
+        let newSpeaking = false;
+
+        for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
+            const volume = this.speakingVolumeSamples[i];
+
+            if (volume > this.speakingThreshold) {
+                newSpeaking = true;
+                break;
+            }
+        }
+
        if (this.speaking !== newSpeaking) {
            this.speaking = newSpeaking;
            this.emit(CallFeedEvent.Speaking, this.speaking);
        }

-            this.volumeLooper();
-        }, POLLING_INTERVAL);
-    }
+        this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
+    };

    public dispose(): void {
        clearTimeout(this.volumeLooperTimeout);