Port some changes from group calls branch to develop (#2001)

* Add some useful getters

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Add removeLocalFeed()

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Don't updateStream() if they're the same

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Add isSpeaking()

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Improve speaking detection using history

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Make code for placing and answering calls more flexible

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

* Correctly log stream id

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

* Remove mistaken parameter

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

* Add a unit

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

Co-authored-by: Robert Long <robert@robertlong.me>

src/webrtc/call.ts

@@ -416,10 +416,26 @@ export class MatrixCall extends EventEmitter {
         return this.localUsermediaFeed?.stream;
     }
 
-    private get localScreensharingStream(): MediaStream {
+    public get localScreensharingStream(): MediaStream {
         return this.localScreensharingFeed?.stream;
     }
 
+    public get remoteUsermediaFeed(): CallFeed {
+        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
+    }
+
+    public get remoteScreensharingFeed(): CallFeed {
+        return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
+    }
+
+    public get remoteUsermediaStream(): MediaStream {
+        return this.remoteUsermediaFeed?.stream;
+    }
+
+    public get remoteScreensharingStream(): MediaStream {
+        return this.remoteScreensharingFeed?.stream;
+    }
+
     private getFeedByStreamId(streamId: string): CallFeed {
         return this.getFeeds().find((feed) => feed.stream.id === streamId);
     }
@@ -551,51 +567,87 @@ export class MatrixCall extends EventEmitter {
         logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`);
     }
 
-    private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
+    private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
         const userId = this.client.getUserId();
 
+        // TODO: Find out what is going on here
+        // why do we enable audio (and only audio) tracks here? -- matthew
+        setTracksEnabled(stream.getAudioTracks(), true);
+
         // We try to replace an existing feed if there already is one with the same purpose
         const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose);
         if (existingFeed) {
             existingFeed.setNewStream(stream);
         } else {
-            this.feeds.push(new CallFeed({
+            this.pushLocalFeed(
-                client: this.client,
+                new CallFeed({
-                roomId: this.roomId,
+                    client: this.client,
-                audioMuted: stream.getAudioTracks().length === 0,
+                    roomId: this.roomId,
-                videoMuted: stream.getVideoTracks().length === 0,
+                    audioMuted: stream.getAudioTracks().length === 0,
-                userId,
+                    videoMuted: stream.getVideoTracks().length === 0,
-                stream,
+                    userId,
-                purpose,
+                    stream,
-            }));
+                    purpose,
+                }),
+                addToPeerConnection,
+            );
             this.emit(CallEvent.FeedsChanged, this.feeds);
         }
+    }
 
-        // TODO: Find out what is going on here
+    /**
-        // why do we enable audio (and only audio) tracks here? -- matthew
+     * Pushes supplied feed to the call
-        setTracksEnabled(stream.getAudioTracks(), true);
+     * @param {CallFeed} callFeed to push
+     * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection
+     */
+    public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void {
+        this.feeds.push(callFeed);
 
         if (addToPeerConnection) {
-            const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ?
+            const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ?
                 this.usermediaSenders : this.screensharingSenders;
             // Empty the array
             senderArray.splice(0, senderArray.length);
 
-            this.emit(CallEvent.FeedsChanged, this.feeds);
+            for (const track of callFeed.stream.getTracks()) {
-            for (const track of stream.getTracks()) {
                 logger.info(
                     `Adding track (` +
                     `id="${track.id}", ` +
                     `kind="${track.kind}", ` +
-                    `streamId="${stream.id}", ` +
+                    `streamId="${callFeed.stream.id}", ` +
-                    `streamPurpose="${purpose}"` +
+                    `streamPurpose="${callFeed.purpose}"` +
                     `) to peer connection`,
                 );
-                senderArray.push(this.peerConn.addTrack(track, stream));
+                senderArray.push(this.peerConn.addTrack(track, callFeed.stream));
             }
         }
 
-        logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`);
+        logger.info(
+            `Pushed local stream ` +
+            `(id="${callFeed.stream.id}", ` +
+            `active="${callFeed.stream.active}", ` +
+            `purpose="${callFeed.purpose}")`,
+        );
+
+        this.emit(CallEvent.FeedsChanged, this.feeds);
+    }
+
+    /**
+     * Removes local call feed from the call and its tracks from the peer
+     * connection
+     * @param callFeed to remove
+     */
+    public removeLocalFeed(callFeed: CallFeed): void {
+        const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia
+            ? this.usermediaSenders
+            : this.screensharingSenders;
+
+        for (const sender of senderArray) {
+            this.peerConn.removeTrack(sender);
+        }
+        // Empty the array
+        senderArray.splice(0, senderArray.length);
+        this.deleteFeedByStream(callFeed.stream);
     }
 
     private deleteAllFeeds(): void {
@@ -760,11 +812,27 @@ export class MatrixCall extends EventEmitter {
             this.waitForLocalAVStream = true;
 
             try {
-                const mediaStream = await this.client.getMediaHandler().getUserMediaStream(
+                const stream = await this.client.getMediaHandler().getUserMediaStream(
                     answerWithAudio, answerWithVideo,
                 );
                 this.waitForLocalAVStream = false;
-                this.gotUserMediaForAnswer(mediaStream);
+                const usermediaFeed = new CallFeed({
+                    client: this.client,
+                    roomId: this.roomId,
+                    userId: this.client.getUserId(),
+                    stream,
+                    purpose: SDPStreamMetadataPurpose.Usermedia,
+                    audioMuted: stream.getAudioTracks().length === 0,
+                    videoMuted: stream.getVideoTracks().length === 0,
+                });
+
+                const feeds = [usermediaFeed];
+
+                if (this.localScreensharingFeed) {
+                    feeds.push(this.localScreensharingFeed);
+                }
+
+                this.answerWithCallFeeds(feeds);
             } catch (e) {
                 if (answerWithVideo) {
                     // Try to answer without video
@@ -782,6 +850,14 @@ export class MatrixCall extends EventEmitter {
         }
     }
 
+    public answerWithCallFeeds(callFeeds: CallFeed[]): void {
+        if (this.inviteOrAnswerSent) return;
+
+        logger.debug(`Answering call ${this.callId}`);
+
+        this.gotCallFeedsForAnswer(callFeeds);
+    }
+
     /**
      * Replace this call with a new call, e.g. for glare resolution. Used by
      * MatrixClient.
@@ -793,7 +869,7 @@ export class MatrixCall extends EventEmitter {
             newCall.waitForLocalAVStream = true;
         } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) {
             logger.debug("Handing local stream to new call");
-            newCall.gotUserMediaForAnswer(this.localUsermediaStream);
+            newCall.gotCallFeedsForAnswer(this.getLocalFeeds());
         }
         this.successor = newCall;
         this.emit(CallEvent.Replaced, newCall);
@@ -865,7 +941,7 @@ export class MatrixCall extends EventEmitter {
                 if (this.hasLocalUserMediaAudioTrack) return;
                 if (this.hasLocalUserMediaVideoTrack) return;
 
-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
             } else if (upgradeAudio) {
                 if (this.hasLocalUserMediaAudioTrack) return;
 
@@ -931,7 +1007,7 @@ export class MatrixCall extends EventEmitter {
             try {
                 const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                 if (!stream) return false;
-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
                 return true;
             } catch (err) {
                 this.emit(CallEvent.Error,
@@ -973,7 +1049,7 @@ export class MatrixCall extends EventEmitter {
                 });
                 sender.replaceTrack(track);
 
-                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);
+                this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);
 
                 return true;
             } catch (err) {
@@ -1133,13 +1209,9 @@ export class MatrixCall extends EventEmitter {
         setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted);
     }
 
-    /**
+    private gotCallFeedsForInvite(callFeeds: CallFeed[]): void {
-     * Internal
-     * @param {Object} stream
-     */
-    private gotUserMediaForInvite = async (stream: MediaStream): Promise<void> => {
         if (this.successor) {
-            this.successor.gotUserMediaForAnswer(stream);
+            this.successor.gotCallFeedsForAnswer(callFeeds);
             return;
         }
         if (this.callHasEnded()) {
@@ -1147,12 +1219,15 @@ export class MatrixCall extends EventEmitter {
             return;
         }
 
-        this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
+        for (const feed of callFeeds) {
+            this.pushLocalFeed(feed);
+        }
+
         this.setState(CallState.CreateOffer);
 
         logger.debug("gotUserMediaForInvite");
         // Now we wait for the negotiationneeded event
-    };
+    }
 
     private async sendAnswer(): Promise<void> {
         const answerContent = {
@@ -1201,12 +1276,15 @@ export class MatrixCall extends EventEmitter {
         this.sendCandidateQueue();
     }
 
-    private gotUserMediaForAnswer = async (stream: MediaStream): Promise<void> => {
+    private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise<void> {
-        if (this.callHasEnded()) {
+        if (this.callHasEnded()) return;
-            return;
+
+        this.waitForLocalAVStream = false;
+
+        for (const feed of callFeeds) {
+            this.pushLocalFeed(feed);
         }
 
-        this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
         this.setState(CallState.CreateAnswer);
 
         let myAnswer;
@@ -1234,7 +1312,7 @@ export class MatrixCall extends EventEmitter {
             this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
             return;
         }
-    };
+    }
 
     /**
      * Internal
@@ -1976,15 +2054,41 @@ export class MatrixCall extends EventEmitter {
      * @throws if have passed audio=false.
      */
     public async placeCall(audio: boolean, video: boolean): Promise<void> {
-        logger.debug(`placeCall audio=${audio} video=${video}`);
         if (!audio) {
             throw new Error("You CANNOT start a call without audio");
         }
+        this.setState(CallState.WaitLocalMedia);
+
+        try {
+            const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
+            const callFeed = new CallFeed({
+                client: this.client,
+                roomId: this.roomId,
+                userId: this.client.getUserId(),
+                stream,
+                purpose: SDPStreamMetadataPurpose.Usermedia,
+                audioMuted: stream.getAudioTracks().length === 0,
+                videoMuted: stream.getVideoTracks().length === 0,
+            });
+            await this.placeCallWithCallFeeds([callFeed]);
+        } catch (e) {
+            this.getUserMediaFailed(e);
+            return;
+        }
+    }
+
+    /**
+     * Place a call to this room with call feed.
+     * @param {CallFeed[]} callFeeds to use
+     * @throws if you have not specified a listener for 'error' events.
+     * @throws if have passed audio=false.
+     */
+    public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise<void> {
         this.checkForErrorListener();
+        this.direction = CallDirection.Outbound;
+
         // XXX Find a better way to do this
         this.client.callEventHandler.calls.set(this.callId, this);
-        this.setState(CallState.WaitLocalMedia);
-        this.direction = CallDirection.Outbound;
 
         // make sure we have valid turn creds. Unless something's gone wrong, it should
         // poll and keep the credentials valid so this should be instant.
@@ -1996,14 +2100,7 @@ export class MatrixCall extends EventEmitter {
         // create the peer connection now so it can be gathering candidates while we get user
         // media (assuming a candidate pool size is configured)
         this.peerConn = this.createPeerConnection();
-
+        this.gotCallFeedsForInvite(callFeeds);
-        try {
-            const mediaStream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
-            this.gotUserMediaForInvite(mediaStream);
-        } catch (e) {
-            this.getUserMediaFailed(e);
-            return;
-        }
     }
 
     private createPeerConnection(): RTCPeerConnection {

src/webrtc/callFeed.ts

@@ -19,8 +19,9 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes";
 import { MatrixClient } from "../client";
 import { RoomMember } from "../models/room-member";
 
-const POLLING_INTERVAL = 250; // ms
+const POLLING_INTERVAL = 200; // ms
-const SPEAKING_THRESHOLD = -60; // dB
+export const SPEAKING_THRESHOLD = -60; // dB
+const SPEAKING_SAMPLE_COUNT = 8; // samples
 
 export interface ICallFeedOpts {
     client: MatrixClient;
@@ -43,6 +44,7 @@ export class CallFeed extends EventEmitter {
     public stream: MediaStream;
     public userId: string;
     public purpose: SDPStreamMetadataPurpose;
+    public speakingVolumeSamples: number[];
 
     private client: MatrixClient;
     private roomId: string;
@@ -65,6 +67,7 @@ export class CallFeed extends EventEmitter {
         this.purpose = opts.purpose;
         this.audioMuted = opts.audioMuted;
         this.videoMuted = opts.videoMuted;
+        this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
 
         this.updateStream(null, opts.stream);
 
@@ -78,6 +81,8 @@ export class CallFeed extends EventEmitter {
     }
 
     private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
+        if (newStream === oldStream) return;
+
         if (oldStream) {
             oldStream.removeEventListener("addtrack", this.onAddTrack);
             this.measureVolumeActivity(false);
@@ -152,6 +157,10 @@ export class CallFeed extends EventEmitter {
         return this.stream.getVideoTracks().length === 0 || this.videoMuted;
     }
 
+    public isSpeaking(): boolean {
+        return this.speaking;
+    }
+
     /**
      * Replaces the current MediaStream with a new one.
      * This method should be only used by MatrixCall.
@@ -167,6 +176,7 @@ export class CallFeed extends EventEmitter {
      */
     public setAudioMuted(muted: boolean): void {
         this.audioMuted = muted;
+        this.speakingVolumeSamples.fill(-Infinity);
         this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
     }
 
@@ -191,6 +201,7 @@ export class CallFeed extends EventEmitter {
             this.volumeLooper();
         } else {
             this.measuringVolumeActivity = false;
+            this.speakingVolumeSamples.fill(-Infinity);
             this.emit(CallFeedEvent.VolumeChanged, -Infinity);
         }
     }
@@ -199,31 +210,43 @@ export class CallFeed extends EventEmitter {
         this.speakingThreshold = threshold;
     }
 
-    private volumeLooper(): void {
+    private volumeLooper = () => {
         if (!this.analyser) return;
 
-        this.volumeLooperTimeout = setTimeout(() => {
+        if (!this.measuringVolumeActivity) return;
-            if (!this.measuringVolumeActivity) return;
 
-            this.analyser.getFloatFrequencyData(this.frequencyBinCount);
+        this.analyser.getFloatFrequencyData(this.frequencyBinCount);
 
-            let maxVolume = -Infinity;
+        let maxVolume = -Infinity;
-            for (let i = 0; i < this.frequencyBinCount.length; i++) {
+        for (let i = 0; i < this.frequencyBinCount.length; i++) {
-                if (this.frequencyBinCount[i] > maxVolume) {
+            if (this.frequencyBinCount[i] > maxVolume) {
-                    maxVolume = this.frequencyBinCount[i];
+                maxVolume = this.frequencyBinCount[i];
-                }
             }
+        }
 
-            this.emit(CallFeedEvent.VolumeChanged, maxVolume);
+        this.speakingVolumeSamples.shift();
-            const newSpeaking = maxVolume > this.speakingThreshold;
+        this.speakingVolumeSamples.push(maxVolume);
-            if (this.speaking !== newSpeaking) {
+
-                this.speaking = newSpeaking;
+        this.emit(CallFeedEvent.VolumeChanged, maxVolume);
-                this.emit(CallFeedEvent.Speaking, this.speaking);
+
+        let newSpeaking = false;
+
+        for (let i = 0; i < this.speakingVolumeSamples.length; i++) {
+            const volume = this.speakingVolumeSamples[i];
+
+            if (volume > this.speakingThreshold) {
+                newSpeaking = true;
+                break;
             }
+        }
 
-            this.volumeLooper();
+        if (this.speaking !== newSpeaking) {
-        }, POLLING_INTERVAL);
+            this.speaking = newSpeaking;
-    }
+            this.emit(CallFeedEvent.Speaking, this.speaking);
+        }
+
+        this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
+    };
 
     public dispose(): void {
         clearTimeout(this.volumeLooperTimeout);