diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d7968caa2..62bf94510 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -179,23 +179,26 @@ export class MockMediaStream { export class MockMediaDeviceInfo { constructor( - public kind: "audio" | "video", + public kind: "audioinput" | "videoinput" | "audiooutput", ) { } } export class MockMediaHandler { - public userMediaStreams: MediaStream[] = []; - public screensharingStreams: MediaStream[] = []; + public userMediaStreams: MockMediaStream[] = []; + public screensharingStreams: MockMediaStream[] = []; getUserMediaStream(audio: boolean, video: boolean) { const tracks = []; if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - return new MockMediaStream("mock_stream_from_media_handler", tracks); + const stream = new MockMediaStream("mock_stream_from_media_handler", tracks); + this.userMediaStreams.push(stream); + return stream; } stopUserMediaStream() { } hasAudioDevice() { return true; } + hasVideoDevice() { return true; } stopAllStreams() {} } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 4f0e177d7..338e968b1 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -16,7 +16,7 @@ limitations under the License. import { TestClient } from '../../TestClient'; import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call'; -import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; +import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; import { DUMMY_SDP, MockMediaHandler, @@ -25,6 +25,7 @@ import { installWebRTCMocks, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; +import { EventType } from "../../../src"; const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { const callPromise = call.placeVoiceCall(); @@ -34,6 +35,14 @@ const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { + const callPromise = call.placeVideoCall(); + await client.httpBackend.flush(""); + await callPromise; + + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); +}; + describe('Call', function() { let client; let call; @@ -765,4 +774,75 @@ describe('Call', function() { expect(call.pushLocalFeed).toHaveBeenCalled(); }); }); + + describe("muting", () => { + beforeEach(async () => { + call.sendVoipEvent = jest.fn(); + await startVideoCall(client, call); + }); + + describe("sending sdp_stream_metadata_changed events", () => { + it("should send sdp_stream_metadata_changed when muting audio", async () => { + await call.setMicrophoneMuted(true); + expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, + }, + }, + }); + }); + + it("should send sdp_stream_metadata_changed when muting video", async () => { + await call.setLocalVideoMuted(true); + expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: false, + video_muted: true, + }, + }, + }); + }); + }); + + describe("receiving sdp_stream_metadata_changed events", () => { + const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { + const metadata = { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }; + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track1", "video"), + ])); + call.onSDPStreamMetadataChangedReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: metadata, + }), + }); + return metadata; + }; + + it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { + const metadata = setupCall(true, false); + expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); + }); + + it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { + const metadata = setupCall(false, true); + expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); + }); + }); + }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 4bba4a91d..378fc1de8 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -17,8 +17,16 @@ limitations under the License. import { EventType, GroupCallIntent, GroupCallType, MatrixEvent, Room, RoomMember } from '../../../src'; import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; -import { installWebRTCMocks, MockMediaHandler, MockRTCPeerConnection } from '../../test-utils/webrtc'; -import { ReEmitter } from '../../../src/ReEmitter'; +import { + installWebRTCMocks, + MockMediaHandler, + MockMediaStream, + MockMediaStreamTrack, + MockRTCPeerConnection, +} from '../../test-utils/webrtc'; +import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; +import { sleep } from "../../../src/utils"; +import { ReEmitter } from "../../../src/ReEmitter"; import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; import { MediaHandler } from '../../../src/webrtc/mediaHandler'; @@ -31,6 +39,60 @@ const FAKE_SESSION_ID_1 = "alice1"; const FAKE_USER_ID_2 = "@bob:test.dummy"; const FAKE_DEVICE_ID_2 = "@BBBBBB"; const FAKE_SESSION_ID_2 = "bob1"; +const FAKE_STATE_EVENTS = [ + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + }), + getStateKey: () => FAKE_USER_ID_1, + getRoomId: () => FAKE_ROOM_ID, + }, + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: FAKE_DEVICE_ID_2, + feeds: [], + }], + }], + }), + getStateKey: () => FAKE_USER_ID_2, + getRoomId: () => FAKE_ROOM_ID, + }, { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: "user3_device", + feeds: [], + }], + }], + }), + getStateKey: () => "user3", + getRoomId: () => FAKE_ROOM_ID, + }, +]; + +const ONE_HOUR = 1000 * 60 * 60; + +const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise => { + const groupCall = new GroupCall( + cli, + room, + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + FAKE_CONF_ID, + ); + + await groupCall.create(); + await groupCall.enter(); + + return groupCall; +}; class MockCallMatrixClient { public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler; @@ -47,6 +109,7 @@ class MockCallMatrixClient { }; sendStateEvent = jest.fn(); + sendToDevice = jest.fn(); getMediaHandler() { return this.mediaHandler; } @@ -318,4 +381,142 @@ describe('Group Call', function() { } }); }); + + describe("muting", () => { + let mockClient: MatrixClient; + let room: Room; + + beforeEach(() => { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { + return type === EventType.GroupCallMemberPrefix + ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS + : { getContent: () => ([]) }; + }); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + }); + + describe("local muting", () => { + it("should mute local audio when calling setMicrophoneMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc, call) => { + acc.push(...call.localUsermediaStream.getAudioTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setMicrophoneMuted(true); + + groupCall.localCallFeed.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(true, null); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(true, null)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + + it("should mute local video when calling setLocalVideoMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc, call) => { + acc.push(...call.localUsermediaStream.getVideoTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setLocalVideoMuted(true); + + groupCall.localCallFeed.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(null, true); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(null, true)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + }); + + describe("remote muting", () => { + const getMetadataEvent = (audio: boolean, video: boolean): MatrixEvent => ({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }, + }), + } as MatrixEvent); + + it("should mute remote feed's audio after receiving metadata with video audio", async () => { + const metadataEvent = getMetadataEvent(true, false); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee); + expect(feed.isAudioMuted()).toBe(true); + expect(feed.isVideoMuted()).toBe(false); + + groupCall.terminate(); + }); + + it("should mute remote feed's video after receiving metadata with video muted", async () => { + const metadataEvent = getMetadataEvent(false, true); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee); + expect(feed.isAudioMuted()).toBe(false); + expect(feed.isVideoMuted()).toBe(true); + + groupCall.terminate(); + }); + }); + }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 83239fade..879bc49be 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -373,6 +373,11 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.clear(); clearTimeout(this.retryCallLoopTimeout); + for (const [userId] of this.memberStateExpirationTimers) { + clearTimeout(this.memberStateExpirationTimers.get(userId)); + this.memberStateExpirationTimers.delete(userId); + } + if (this.transmitTimer !== null) { clearTimeout(this.transmitTimer); this.transmitTimer = null;