Add group call tests for muting (#2590)

spec/test-utils/webrtc.ts

@@ -179,23 +179,26 @@ export class MockMediaStream {
 
 export class MockMediaDeviceInfo {
     constructor(
-        public kind: "audio" | "video",
+        public kind: "audioinput" | "videoinput" | "audiooutput",
     ) { }
 }
 
 export class MockMediaHandler {
-    public userMediaStreams: MediaStream[] = [];
+    public userMediaStreams: MockMediaStream[] = [];
-    public screensharingStreams: MediaStream[] = [];
+    public screensharingStreams: MockMediaStream[] = [];
 
     getUserMediaStream(audio: boolean, video: boolean) {
         const tracks = [];
         if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
         if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
 
-        return new MockMediaStream("mock_stream_from_media_handler", tracks);
+        const stream = new MockMediaStream("mock_stream_from_media_handler", tracks);
+        this.userMediaStreams.push(stream);
+        return stream;
     }
     stopUserMediaStream() { }
     hasAudioDevice() { return true; }
+    hasVideoDevice() { return true; }
     stopAllStreams() {}
 }
 

spec/unit/webrtc/call.spec.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import { TestClient } from '../../TestClient';
 import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call';
-import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes';
+import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes';
 import {
     DUMMY_SDP,
     MockMediaHandler,
@@ -25,6 +25,7 @@ import {
     installWebRTCMocks,
 } from "../../test-utils/webrtc";
 import { CallFeed } from "../../../src/webrtc/callFeed";
+import { EventType } from "../../../src";
 
 const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise<void> => {
     const callPromise = call.placeVoiceCall();
@@ -34,6 +35,14 @@ const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise<voi
     call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
 };
 
+const startVideoCall = async (client: TestClient, call: MatrixCall): Promise<void> => {
+    const callPromise = call.placeVideoCall();
+    await client.httpBackend.flush("");
+    await callPromise;
+
+    call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
+};
+
 describe('Call', function() {
     let client;
     let call;
@@ -765,4 +774,75 @@ describe('Call', function() {
             expect(call.pushLocalFeed).toHaveBeenCalled();
         });
     });
+
+    describe("muting", () => {
+        beforeEach(async () => {
+            call.sendVoipEvent = jest.fn();
+            await startVideoCall(client, call);
+        });
+
+        describe("sending sdp_stream_metadata_changed events", () => {
+            it("should send sdp_stream_metadata_changed when muting audio", async () => {
+                await call.setMicrophoneMuted(true);
+                expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, {
+                    [SDPStreamMetadataKey]: {
+                        mock_stream_from_media_handler: {
+                            purpose: SDPStreamMetadataPurpose.Usermedia,
+                            audio_muted: true,
+                            video_muted: false,
+                        },
+                    },
+                });
+            });
+
+            it("should send sdp_stream_metadata_changed when muting video", async () => {
+                await call.setLocalVideoMuted(true);
+                expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, {
+                    [SDPStreamMetadataKey]: {
+                        mock_stream_from_media_handler: {
+                            purpose: SDPStreamMetadataPurpose.Usermedia,
+                            audio_muted: false,
+                            video_muted: true,
+                        },
+                    },
+                });
+            });
+        });
+
+        describe("receiving sdp_stream_metadata_changed events", () => {
+            const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => {
+                const metadata = {
+                    stream: {
+                        purpose: SDPStreamMetadataPurpose.Usermedia,
+                        audio_muted: audio,
+                        video_muted: video,
+                    },
+                };
+                call.pushRemoteFeed(new MockMediaStream("stream", [
+                    new MockMediaStreamTrack("track1", "audio"),
+                    new MockMediaStreamTrack("track1", "video"),
+                ]));
+                call.onSDPStreamMetadataChangedReceived({
+                    getContent: () => ({
+                        [SDPStreamMetadataKey]: metadata,
+                    }),
+                });
+                return metadata;
+            };
+
+            it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => {
+                const metadata = setupCall(true, false);
+                expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata);
+                expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true);
+                expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false);
+            });
+
+            it("should handle incoming sdp_stream_metadata_changed with video muted", async () => {
+                const metadata = setupCall(false, true);
+                expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata);
+                expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false);
+                expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true);
+            });
+        });
+    });
 });

spec/unit/webrtc/groupCall.spec.ts

@@ -17,8 +17,16 @@ limitations under the License.
 import { EventType, GroupCallIntent, GroupCallType, MatrixEvent, Room, RoomMember } from '../../../src';
 import { GroupCall } from "../../../src/webrtc/groupCall";
 import { MatrixClient } from "../../../src/client";
-import { installWebRTCMocks, MockMediaHandler, MockRTCPeerConnection } from '../../test-utils/webrtc';
+import {
-import { ReEmitter } from '../../../src/ReEmitter';
+    installWebRTCMocks,
+    MockMediaHandler,
+    MockMediaStream,
+    MockMediaStreamTrack,
+    MockRTCPeerConnection,
+} from '../../test-utils/webrtc';
+import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes";
+import { sleep } from "../../../src/utils";
+import { ReEmitter } from "../../../src/ReEmitter";
 import { TypedEventEmitter } from '../../../src/models/typed-event-emitter';
 import { MediaHandler } from '../../../src/webrtc/mediaHandler';
 
@@ -31,6 +39,60 @@ const FAKE_SESSION_ID_1 = "alice1";
 const FAKE_USER_ID_2 = "@bob:test.dummy";
 const FAKE_DEVICE_ID_2 = "@BBBBBB";
 const FAKE_SESSION_ID_2 = "bob1";
+const FAKE_STATE_EVENTS = [
+    {
+        getContent: () => ({
+            ["m.expires_ts"]: Date.now() + ONE_HOUR,
+        }),
+        getStateKey: () => FAKE_USER_ID_1,
+        getRoomId: () => FAKE_ROOM_ID,
+    },
+    {
+        getContent: () => ({
+            ["m.expires_ts"]: Date.now() + ONE_HOUR,
+            ["m.calls"]: [{
+                ["m.call_id"]: FAKE_CONF_ID,
+                ["m.devices"]: [{
+                    device_id: FAKE_DEVICE_ID_2,
+                    feeds: [],
+                }],
+            }],
+        }),
+        getStateKey: () => FAKE_USER_ID_2,
+        getRoomId: () => FAKE_ROOM_ID,
+    }, {
+        getContent: () => ({
+            ["m.expires_ts"]: Date.now() + ONE_HOUR,
+            ["m.calls"]: [{
+                ["m.call_id"]: FAKE_CONF_ID,
+                ["m.devices"]: [{
+                    device_id: "user3_device",
+                    feeds: [],
+                }],
+            }],
+        }),
+        getStateKey: () => "user3",
+        getRoomId: () => FAKE_ROOM_ID,
+    },
+];
+
+const ONE_HOUR = 1000 * 60 * 60;
+
+const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise<GroupCall> => {
+    const groupCall = new GroupCall(
+        cli,
+        room,
+        GroupCallType.Video,
+        false,
+        GroupCallIntent.Prompt,
+        FAKE_CONF_ID,
+    );
+
+    await groupCall.create();
+    await groupCall.enter();
+
+    return groupCall;
+};
 
 class MockCallMatrixClient {
     public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler;
@@ -47,6 +109,7 @@ class MockCallMatrixClient {
     };
 
     sendStateEvent = jest.fn();
+    sendToDevice = jest.fn();
 
     getMediaHandler() { return this.mediaHandler; }
 
@@ -318,4 +381,142 @@ describe('Group Call', function() {
             }
         });
     });
+
+    describe("muting", () => {
+        let mockClient: MatrixClient;
+        let room: Room;
+
+        beforeEach(() => {
+            const typedMockClient = new MockCallMatrixClient(
+                FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1,
+            );
+            mockClient = typedMockClient as unknown as MatrixClient;
+
+            room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1);
+            room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => {
+                return type === EventType.GroupCallMemberPrefix
+                    ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS
+                    : { getContent: () => ([]) };
+            });
+            room.getMember = jest.fn().mockImplementation((userId) => ({ userId }));
+        });
+
+        describe("local muting", () => {
+            it("should mute local audio when calling setMicrophoneMuted()", async () => {
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+
+                groupCall.localCallFeed.setAudioVideoMuted = jest.fn();
+                const setAVMutedArray = groupCall.calls.map(call => {
+                    call.localUsermediaFeed.setAudioVideoMuted = jest.fn();
+                    return call.localUsermediaFeed.setAudioVideoMuted;
+                });
+                const tracksArray = groupCall.calls.reduce((acc, call) => {
+                    acc.push(...call.localUsermediaStream.getAudioTracks());
+                    return acc;
+                }, []);
+                const sendMetadataUpdateArray = groupCall.calls.map(call => {
+                    call.sendMetadataUpdate = jest.fn();
+                    return call.sendMetadataUpdate;
+                });
+
+                await groupCall.setMicrophoneMuted(true);
+
+                groupCall.localCallFeed.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false));
+                expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(true, null);
+                setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(true, null));
+                tracksArray.forEach(track => expect(track.enabled).toBe(false));
+                sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled());
+
+                groupCall.terminate();
+            });
+
+            it("should mute local video when calling setLocalVideoMuted()", async () => {
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+
+                groupCall.localCallFeed.setAudioVideoMuted = jest.fn();
+                const setAVMutedArray = groupCall.calls.map(call => {
+                    call.localUsermediaFeed.setAudioVideoMuted = jest.fn();
+                    return call.localUsermediaFeed.setAudioVideoMuted;
+                });
+                const tracksArray = groupCall.calls.reduce((acc, call) => {
+                    acc.push(...call.localUsermediaStream.getVideoTracks());
+                    return acc;
+                }, []);
+                const sendMetadataUpdateArray = groupCall.calls.map(call => {
+                    call.sendMetadataUpdate = jest.fn();
+                    return call.sendMetadataUpdate;
+                });
+
+                await groupCall.setLocalVideoMuted(true);
+
+                groupCall.localCallFeed.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false));
+                expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(null, true);
+                setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(null, true));
+                tracksArray.forEach(track => expect(track.enabled).toBe(false));
+                sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled());
+
+                groupCall.terminate();
+            });
+        });
+
+        describe("remote muting", () => {
+            const getMetadataEvent = (audio: boolean, video: boolean): MatrixEvent => ({
+                getContent: () => ({
+                    [SDPStreamMetadataKey]: {
+                        stream: {
+                            purpose: SDPStreamMetadataPurpose.Usermedia,
+                            audio_muted: audio,
+                            video_muted: video,
+                        },
+                    },
+                }),
+            } as MatrixEvent);
+
+            it("should mute remote feed's audio after receiving metadata with video audio", async () => {
+                const metadataEvent = getMetadataEvent(true, false);
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+
+                // It takes a bit of time for the calls to get created
+                await sleep(10);
+
+                const call = groupCall.calls[0];
+                call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
+                // @ts-ignore Mock
+                call.pushRemoteFeed(new MockMediaStream("stream", [
+                    new MockMediaStreamTrack("audio_track", "audio"),
+                    new MockMediaStreamTrack("video_track", "video"),
+                ]));
+                call.onSDPStreamMetadataChangedReceived(metadataEvent);
+
+                const feed = groupCall.getUserMediaFeedByUserId(call.invitee);
+                expect(feed.isAudioMuted()).toBe(true);
+                expect(feed.isVideoMuted()).toBe(false);
+
+                groupCall.terminate();
+            });
+
+            it("should mute remote feed's video after receiving metadata with video muted", async () => {
+                const metadataEvent = getMetadataEvent(false, true);
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+
+                // It takes a bit of time for the calls to get created
+                await sleep(10);
+
+                const call = groupCall.calls[0];
+                call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
+                // @ts-ignore Mock
+                call.pushRemoteFeed(new MockMediaStream("stream", [
+                    new MockMediaStreamTrack("audio_track", "audio"),
+                    new MockMediaStreamTrack("video_track", "video"),
+                ]));
+                call.onSDPStreamMetadataChangedReceived(metadataEvent);
+
+                const feed = groupCall.getUserMediaFeedByUserId(call.invitee);
+                expect(feed.isAudioMuted()).toBe(false);
+                expect(feed.isVideoMuted()).toBe(true);
+
+                groupCall.terminate();
+            });
+        });
+    });
 });

src/webrtc/groupCall.ts

@@ -373,6 +373,11 @@ export class GroupCall extends TypedEventEmitter<
         this.retryCallCounts.clear();
         clearTimeout(this.retryCallLoopTimeout);
 
+        for (const [userId] of this.memberStateExpirationTimers) {
+            clearTimeout(this.memberStateExpirationTimers.get(userId));
+            this.memberStateExpirationTimers.delete(userId);
+        }
+
         if (this.transmitTimer !== null) {
             clearTimeout(this.transmitTimer);
             this.transmitTimer = null;