Merge pull request #1812 from SimonBrandner/feature/muting

Support for MSC3291: Muting in VoIP calls

spec/unit/utils.spec.ts

@@ -493,4 +493,68 @@ describe("utils", function() {
             expect(deepSortedObjectEntries(input)).toMatchObject(output);
         });
     });
+
+    describe("recursivelyAssign", () => {
+        it("doesn't override with null/undefined", () => {
+            const result = utils.recursivelyAssign(
+                {
+                    string: "Hello world",
+                    object: {},
+                    float: 0.1,
+                }, {
+                    string: null,
+                    object: undefined,
+                },
+                true,
+            );
+
+            expect(result).toStrictEqual({
+                string: "Hello world",
+                object: {},
+                float: 0.1,
+            });
+        });
+
+        it("assigns recursively", () => {
+            const result = utils.recursivelyAssign(
+                {
+                    number: 42,
+                    object: {
+                        message: "Hello world",
+                        day: "Monday",
+                        langs: {
+                            compiled: ["c++"],
+                        },
+                    },
+                    thing: "string",
+                }, {
+                    number: 2,
+                    object: {
+                        message: "How are you",
+                        day: "Friday",
+                        langs: {
+                            compiled: ["c++", "c"],
+                        },
+                    },
+                    thing: {
+                        aSubThing: "something",
+                    },
+                },
+            );
+
+            expect(result).toStrictEqual({
+                number: 2,
+                object: {
+                    message: "How are you",
+                    day: "Friday",
+                    langs: {
+                        compiled: ["c++", "c"],
+                    },
+                },
+                thing: {
+                    aSubThing: "something",
+                },
+            });
+        });
+    });
 });

spec/unit/webrtc/call.spec.ts

@@ -321,16 +321,19 @@ describe('Call', function() {
                     [SDPStreamMetadataKey]: {
                         "stream_id": {
                             purpose: SDPStreamMetadataPurpose.Usermedia,
+                            audio_muted: true,
+                            video_muted: false,
                         },
                     },
                 };
             },
         });
 
-        call.pushRemoteFeed({ id: "stream_id" });
+        call.pushRemoteFeed({ id: "stream_id", getAudioTracks: () => ["track1"], getVideoTracks: () => ["track1"] });
-        expect(call.getFeeds().find((feed) => {
+        const feed = call.getFeeds().find((feed) => feed.stream.id === "stream_id");
-            return feed.stream.id === "stream_id";
+        expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia);
-        })?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia);
+        expect(feed?.isAudioMuted()).toBeTruthy();
+        expect(feed?.isVideoMuted()).not.toBeTruthy();
     });
 
     it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => {

src/@types/event.ts

@@ -53,6 +53,8 @@ export enum EventType {
     CallReject = "m.call.reject",
     CallSelectAnswer = "m.call.select_answer",
     CallNegotiate = "m.call.negotiate",
+    CallSDPStreamMetadataChanged = "m.call.sdp_stream_metadata_changed",
+    CallSDPStreamMetadataChangedPrefix = "org.matrix.call.sdp_stream_metadata_changed",
     CallReplaces = "m.call.replaces",
     CallAssertedIdentity = "m.call.asserted_identity",
     CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity",

src/utils.ts

@@ -694,3 +694,25 @@ const collator = new Intl.Collator();
 export function compare(a: string, b: string): number {
     return collator.compare(a, b);
 }
+
+/**
+ * This function is similar to Object.assign() but it assigns recursively and
+ * allows you to ignore nullish values from the source
+ *
+ * @param {Object} target
+ * @param {Object} source
+ * @returns the target object
+ */
+export function recursivelyAssign(target: Object, source: Object, ignoreNullish = false): any {
+    for (const [sourceKey, sourceValue] of Object.entries(source)) {
+        if (target[sourceKey] instanceof Object && sourceValue) {
+            recursivelyAssign(target[sourceKey], sourceValue);
+            continue;
+        }
+        if ((sourceValue !== null && sourceValue !== undefined) || !ignoreNullish) {
+            target[sourceKey] = sourceValue;
+            continue;
+        }
+    }
+    return target;
+}

src/webrtc/call.ts

@@ -36,6 +36,7 @@ import {
     SDPStreamMetadataPurpose,
     SDPStreamMetadata,
     SDPStreamMetadataKey,
+    MCallSDPStreamMetadataChanged,
 } from './callEventTypes';
 import { CallFeed } from './callFeed';
 
@@ -353,8 +354,6 @@ export class MatrixCall extends EventEmitter {
         this.makingOffer = false;
 
         this.remoteOnHold = false;
-        this.micMuted = false;
-        this.vidMuted = false;
 
         this.feeds = [];
 
@@ -402,6 +401,14 @@ export class MatrixCall extends EventEmitter {
         return this.remoteAssertedIdentity;
     }
 
+    public get localUsermediaFeed(): CallFeed {
+        return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
+    }
+
+    private getFeedByStreamId(streamId: string): CallFeed {
+        return this.getFeeds().find((feed) => feed.stream.id === streamId);
+    }
+
     /**
      * Returns an array of all CallFeeds
      * @returns {Array<CallFeed>} CallFeeds
@@ -431,10 +438,12 @@ export class MatrixCall extends EventEmitter {
      * @returns {SDPStreamMetadata} localSDPStreamMetadata
      */
     private getLocalSDPStreamMetadata(): SDPStreamMetadata {
-        const metadata = {};
+        const metadata: SDPStreamMetadata = {};
         for (const localFeed of this.getLocalFeeds()) {
             metadata[localFeed.stream.id] = {
                 purpose: localFeed.purpose,
+                audio_muted: localFeed.isAudioMuted(),
+                video_muted: localFeed.isVideoMuted(),
             };
         }
         logger.debug("Got local SDPStreamMetadata", metadata);
@@ -459,6 +468,8 @@ export class MatrixCall extends EventEmitter {
 
         const userId = this.getOpponentMember().userId;
         const purpose = this.remoteSDPStreamMetadata[stream.id].purpose;
+        const audioMuted = this.remoteSDPStreamMetadata[stream.id].audio_muted;
+        const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted;
 
         if (!purpose) {
             logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`);
@@ -471,7 +482,7 @@ export class MatrixCall extends EventEmitter {
         if (existingFeed) {
             existingFeed.setNewStream(stream);
         } else {
-            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId));
+            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, audioMuted, videoMuted));
             this.emit(CallEvent.FeedsChanged, this.feeds);
         }
 
@@ -498,11 +509,11 @@ export class MatrixCall extends EventEmitter {
 
         // Try to find a feed with the same stream id as the new stream,
         // if we find it replace the old stream with the new one
-        const feed = this.feeds.find((feed) => feed.stream.id === stream.id);
+        const feed = this.getFeedByStreamId(stream.id);
         if (feed) {
             feed.setNewStream(stream);
         } else {
-            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId));
+            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false));
             this.emit(CallEvent.FeedsChanged, this.feeds);
         }
 
@@ -517,7 +528,7 @@ export class MatrixCall extends EventEmitter {
         if (existingFeed) {
             existingFeed.setNewStream(stream);
         } else {
-            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId));
+            this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false));
             this.emit(CallEvent.FeedsChanged, this.feeds);
         }
 
@@ -555,7 +566,7 @@ export class MatrixCall extends EventEmitter {
     private deleteFeedByStream(stream: MediaStream) {
         logger.debug(`Removing feed with stream id ${stream.id}`);
 
-        const feed = this.feeds.find((feed) => feed.stream.id === stream.id);
+        const feed = this.getFeedByStreamId(stream.id);
         if (!feed) {
             logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`);
             return;
@@ -605,7 +616,7 @@ export class MatrixCall extends EventEmitter {
 
         const sdpStreamMetadata = invite[SDPStreamMetadataKey];
         if (sdpStreamMetadata) {
-            this.remoteSDPStreamMetadata = sdpStreamMetadata;
+            this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
         } else {
             logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams");
         }
@@ -891,7 +902,7 @@ export class MatrixCall extends EventEmitter {
      * @param {boolean} muted True to mute the outbound video.
      */
     setLocalVideoMuted(muted: boolean) {
-        this.vidMuted = muted;
+        this.localUsermediaFeed?.setVideoMuted(muted);
         this.updateMuteStatus();
     }
 
@@ -905,8 +916,7 @@ export class MatrixCall extends EventEmitter {
      * (including if the call is not set up yet).
      */
     isLocalVideoMuted(): boolean {
-        if (this.type === CallType.Voice) return true;
+        return this.localUsermediaFeed?.isVideoMuted();
-        return this.vidMuted;
     }
 
     /**
@@ -914,7 +924,7 @@ export class MatrixCall extends EventEmitter {
      * @param {boolean} muted True to mute the mic.
      */
     setMicrophoneMuted(muted: boolean) {
-        this.micMuted = muted;
+        this.localUsermediaFeed?.setAudioMuted(muted);
         this.updateMuteStatus();
     }
 
@@ -928,7 +938,7 @@ export class MatrixCall extends EventEmitter {
      * is not set up yet).
      */
     isMicrophoneMuted(): boolean {
-        return this.micMuted;
+        return this.localUsermediaFeed?.isAudioMuted();
     }
 
     /**
@@ -991,14 +1001,14 @@ export class MatrixCall extends EventEmitter {
     }
 
     private updateMuteStatus() {
-        if (!this.localAVStream) {
+        this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, {
-            return;
+            [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(),
-        }
+        });
+
+        const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold;
+        const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold;
 
-        const micShouldBeMuted = this.micMuted || this.remoteOnHold;
         setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted);
-
-        const vidShouldBeMuted = this.vidMuted || this.remoteOnHold;
         setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted);
     }
 
@@ -1214,7 +1224,7 @@ export class MatrixCall extends EventEmitter {
 
         const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey];
         if (sdpStreamMetadata) {
-            this.remoteSDPStreamMetadata = sdpStreamMetadata;
+            this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
         } else {
             logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams");
         }
@@ -1289,9 +1299,9 @@ export class MatrixCall extends EventEmitter {
 
         const prevLocalOnHold = this.isLocalOnHold();
 
-        const metadata = event.getContent()[SDPStreamMetadataKey];
+        const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey];
-        if (metadata) {
+        if (sdpStreamMetadata) {
-            this.remoteSDPStreamMetadata = metadata;
+            this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
         } else {
             logger.warn("Received negotiation event without SDPStreamMetadata!");
         }
@@ -1321,6 +1331,22 @@ export class MatrixCall extends EventEmitter {
         }
     }
 
+    private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void {
+        this.remoteSDPStreamMetadata = utils.recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
+        for (const feed of this.getRemoteFeeds()) {
+            const streamId = feed.stream.id;
+            feed.setAudioMuted(this.remoteSDPStreamMetadata[streamId]?.audio_muted);
+            feed.setVideoMuted(this.remoteSDPStreamMetadata[streamId]?.video_muted);
+            feed.purpose = this.remoteSDPStreamMetadata[streamId]?.purpose;
+        }
+    }
+
+    public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void {
+        const content = event.getContent<MCallSDPStreamMetadataChanged>();
+        const metadata = content[SDPStreamMetadataKey];
+        this.updateRemoteSDPStreamMetadata(metadata);
+    }
+
     async onAssertedIdentityReceived(event: MatrixEvent) {
         if (!event.getContent().asserted_identity) return;
 

src/webrtc/callEventHandler.ts

@@ -297,6 +297,18 @@ export class CallEventHandler {
             }
 
             call.onAssertedIdentityReceived(event);
+        } else if (
+            event.getType() === EventType.CallSDPStreamMetadataChanged ||
+            event.getType() === EventType.CallSDPStreamMetadataChangedPrefix
+        ) {
+            if (!call) return;
+
+            if (event.getContent().party_id === call.ourPartyId) {
+                // Ignore remote echo
+                return;
+            }
+
+            call.onSDPStreamMetadataChangedReceived(event);
         }
     }
 }

src/webrtc/callEventTypes.ts

@@ -11,6 +11,8 @@ export enum SDPStreamMetadataPurpose {
 
 export interface SDPStreamMetadataObject {
     purpose: SDPStreamMetadataPurpose;
+    audio_muted: boolean;
+    video_muted: boolean;
 }
 
 export interface SDPStreamMetadata {
@@ -41,6 +43,10 @@ export interface MCallOfferNegotiate {
     [SDPStreamMetadataKey]: SDPStreamMetadata;
 }
 
+export interface MCallSDPStreamMetadataChanged {
+    [SDPStreamMetadataKey]: SDPStreamMetadata;
+}
+
 export interface MCallReplacesTarget {
     id: string;
     display_name: string;

src/webrtc/callFeed.ts

@@ -21,6 +21,7 @@ import { RoomMember } from "../models/room-member";
 
 export enum CallFeedEvent {
     NewStream = "new_stream",
+    MuteStateChanged = "mute_state_changed"
 }
 
 export class CallFeed extends EventEmitter {
@@ -30,6 +31,8 @@ export class CallFeed extends EventEmitter {
         public purpose: SDPStreamMetadataPurpose,
         private client: MatrixClient,
         private roomId: string,
+        private audioMuted: boolean,
+        private videoMuted: boolean,
     ) {
         super();
     }
@@ -51,15 +54,13 @@ export class CallFeed extends EventEmitter {
         return this.userId === this.client.getUserId();
     }
 
-    // TODO: The two following methods should be later replaced
-    // by something that will also check if the remote is muted
     /**
      * Returns true if audio is muted or if there are no audio
      * tracks, otherwise returns false
      * @returns {boolean} is audio muted?
      */
     public isAudioMuted(): boolean {
-        return this.stream.getAudioTracks().length === 0;
+        return this.stream.getAudioTracks().length === 0 || this.audioMuted;
     }
 
     /**
@@ -69,7 +70,7 @@ export class CallFeed extends EventEmitter {
      */
     public isVideoMuted(): boolean {
         // We assume only one video track
-        return this.stream.getVideoTracks().length === 0;
+        return this.stream.getVideoTracks().length === 0 || this.videoMuted;
     }
 
     /**
@@ -81,4 +82,14 @@ export class CallFeed extends EventEmitter {
         this.stream = newStream;
         this.emit(CallFeedEvent.NewStream, this.stream);
     }
+
+    public setAudioMuted(muted: boolean): void {
+        this.audioMuted = muted;
+        this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
+    }
+
+    public setVideoMuted(muted: boolean): void {
+        this.videoMuted = muted;
+        this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
+    }
 }