diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 4dd7e1633..fb8107aff 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -160,10 +160,13 @@ export class MockRTCRtpSender { export class MockMediaStreamTrack { constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } - stop() { } + stop = jest.fn(); listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; + public settings: MediaTrackSettings; + + getSettings(): MediaTrackSettings { return this.settings; } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation @@ -181,6 +184,8 @@ export class MockMediaStreamTrack { return t !== eventType || c !== callback; }); } + + typed(): MediaStreamTrack { return this as unknown as MediaStreamTrack; } } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own @@ -217,8 +222,12 @@ export class MockMediaStream { } removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } - clone() { - return new MockMediaStream(this.id, this.tracks); + clone(): MediaStream { + return new MockMediaStream(this.id + ".clone", this.tracks).typed(); + } + + isCloneOf(stream: MediaStream) { + return this.id === stream.id + ".clone"; } // syntactic sugar for typing @@ -231,6 +240,8 @@ export class MockMediaDeviceInfo { constructor( public kind: "audioinput" | "videoinput" | "audiooutput", ) { } + + typed(): MediaDeviceInfo { return this as unknown as MediaDeviceInfo; } } export class MockMediaHandler { @@ -267,15 +278,27 @@ export class MockMediaHandler { typed(): MediaHandler { return this as unknown as MediaHandler; } } +export class MockMediaDevices { + enumerateDevices = jest.fn, []>().mockResolvedValue([ + new MockMediaDeviceInfo("audioinput").typed(), + new MockMediaDeviceInfo("videoinput").typed(), + ]); + + getUserMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_stream").typed()), + ); + + getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_display_stream").typed()), + ); + + typed(): MediaDevices { return this as unknown as MediaDevices; } +} + export function installWebRTCMocks() { global.navigator = { - mediaDevices: { - // @ts-ignore Mock - getUserMedia: () => new MockMediaStream("local_stream"), - // @ts-ignore Mock - enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], - }, - }; + mediaDevices: new MockMediaDevices().typed(), + } as unknown as Navigator; global.window = { // @ts-ignore Mock diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts new file mode 100644 index 000000000..94396845c --- /dev/null +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -0,0 +1,445 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { GroupCall, MatrixCall, MatrixClient } from "../../../src"; +import { MediaHandler, MediaHandlerEvent } from "../../../src/webrtc/mediaHandler"; +import { MockMediaDeviceInfo, MockMediaDevices, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; + +const FAKE_AUDIO_INPUT_ID = "aaaaaaaa"; +const FAKE_VIDEO_INPUT_ID = "vvvvvvvv"; +const FAKE_DESKTOP_SOURCE_ID = "ddddddd"; + +describe('Media Handler', function() { + let mockMediaDevices: MockMediaDevices; + let mediaHandler: MediaHandler; + let calls: Map; + let groupCalls: Map; + + beforeEach(() => { + mockMediaDevices = new MockMediaDevices(); + + global.navigator = { + mediaDevices: mockMediaDevices.typed(), + } as unknown as Navigator; + + calls = new Map(); + groupCalls = new Map(); + + mediaHandler = new MediaHandler({ + callEventHandler: { + calls, + }, + groupCallEventHandler: { + groupCalls, + }, + } as unknown as MatrixClient); + }); + + it("does not trigger update after restore media settings ", () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("sets device IDs on restore media settings", async () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets audio device ID", async () => { + await mediaHandler.setAudioInput(FAKE_AUDIO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, false); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + })); + }); + + it("sets video device ID", async () => { + await mediaHandler.setVideoInput(FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(false, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets media inputs", async () => { + await mediaHandler.setMediaInputs(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + describe("updateLocalUsermediaStreams", () => { + let localStreamsChangedHandler: jest.Mock; + + beforeEach(() => { + localStreamsChangedHandler = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + afterEach(() => { + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + it("does nothing if it has no streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("does not emit LocalStreamsChanged if it had no streams", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).not.toHaveBeenCalled(); + }); + + describe("with existing streams", () => { + let stopTrack: jest.Mock; + let updateLocalUsermediaStream: jest.Mock; + + beforeEach(() => { + stopTrack = jest.fn(); + + mediaHandler.userMediaStreams = [ + { + getTracks: () => [{ + stop: stopTrack, + } as unknown as MediaStreamTrack], + } as unknown as MediaStream, + ]; + + updateLocalUsermediaStream = jest.fn(); + }); + + it("stops existing streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(stopTrack).toHaveBeenCalled(); + }); + + it("replaces streams on calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(false), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on ended calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(true), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("replaces streams on group calls", async () => { + groupCalls.set("some_group_call", { + localCallFeed: {}, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on group calls with no localCallFeed", async () => { + groupCalls.set("some_group_call", { + localCallFeed: null, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("emits LocalStreamsChanged", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).toHaveBeenCalled(); + }); + }); + }); + + describe("hasAudioDevice", () => { + it("returns true if the system has audio inputs", async () => { + expect(await mediaHandler.hasAudioDevice()).toEqual(true); + }); + + it("returns false if the system has no audio inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("videoinput").typed(), + ])); + expect(await mediaHandler.hasAudioDevice()).toEqual(false); + }); + }); + + describe("hasVideoDevice", () => { + it("returns true if the system has video inputs", async () => { + expect(await mediaHandler.hasVideoDevice()).toEqual(true); + }); + + it("returns false if the system has no video inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("audioinput").typed(), + ])); + expect(await mediaHandler.hasVideoDevice()).toEqual(false); + }); + }); + + describe("getUserMediaStream", () => { + beforeEach(() => { + // replace this with one that returns a new object each time so we can + // tell whether we've ended up with the same stream + mockMediaDevices.getUserMedia.mockImplementation((constraints: MediaStreamConstraints) => { + const stream = new MockMediaStream("local_stream"); + if (constraints.audio) { + const track = new MockMediaStreamTrack("audio_track", "audio"); + track.settings = { deviceId: FAKE_AUDIO_INPUT_ID }; + stream.addTrack(track); + } + if (constraints.video) { + const track = new MockMediaStreamTrack("video_track", "video"); + track.settings = { deviceId: FAKE_VIDEO_INPUT_ID }; + stream.addTrack(track); + } + + return Promise.resolve(stream.typed()); + }); + + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + }); + + it("returns the same stream for reusable streams", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + }); + + it("doesn't re-use stream if reusable is false", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks audio", async () => { + const stream1 = await mediaHandler.getUserMediaStream(false, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks video", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("strips unwanted audio tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getAudioTracks().length).toEqual(0); + }); + + it("strips unwanted video tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getVideoTracks().length).toEqual(0); + }); + }); + + describe("getScreensharingStream", () => { + it("gets any screen sharing stream when called with no args", async () => { + const stream = await mediaHandler.getScreensharingStream(); + expect(stream).toBeTruthy(); + expect(stream.getTracks()).toBeTruthy(); + }); + + it("re-uses streams", async () => { + const stream = await mediaHandler.getScreensharingStream(undefined, true); + + expect(mockMediaDevices.getDisplayMedia).toHaveBeenCalled(); + mockMediaDevices.getDisplayMedia.mockClear(); + + const stream2 = await mediaHandler.getScreensharingStream() as unknown as MockMediaStream; + + expect(mockMediaDevices.getDisplayMedia).not.toHaveBeenCalled(); + + expect(stream2.isCloneOf(stream)).toEqual(true); + }); + + it("passes through desktopCapturerSourceId for Electron", async () => { + await mediaHandler.getScreensharingStream({ + desktopCapturerSourceId: FAKE_DESKTOP_SOURCE_ID, + }); + + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: { + mandatory: expect.objectContaining({ + chromeMediaSource: "desktop", + chromeMediaSourceId: FAKE_DESKTOP_SOURCE_ID, + }), + }, + })); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + await mediaHandler.getScreensharingStream(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getUserMediaStream(true, false); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopUserMediaStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(stream); + mediaHandler.stopUserMediaStream(stream); + expect(mediaHandler.userMediaStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopUserMediaStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopScreensharingStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.screensharingStreams).toContain(stream); + mediaHandler.stopScreensharingStream(stream); + expect(mediaHandler.screensharingStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopScreensharingStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopAllStreams", () => { + let userMediaStream: MediaStream; + let screenSharingStream: MediaStream; + + beforeEach(async () => { + userMediaStream = await mediaHandler.getUserMediaStream(true, false); + screenSharingStream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockUserMediaTrack = new MockMediaStreamTrack("audio_track", "audio"); + userMediaStream.addTrack(mockUserMediaTrack.typed()); + + const mockScreenshareTrack = new MockMediaStreamTrack("audio_track", "audio"); + screenSharingStream.addTrack(mockScreenshareTrack.typed()); + + mediaHandler.stopAllStreams(); + + expect(mockUserMediaTrack.stop).toHaveBeenCalled(); + expect(mockScreenshareTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).toContain(screenSharingStream); + mediaHandler.stopAllStreams(); + expect(mediaHandler.userMediaStreams).not.toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).not.toContain(screenSharingStream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopAllStreams(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); +}); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 1b55cfbb9..2eba5f2f5 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -179,13 +179,31 @@ export class MediaHandler extends TypedEventEmitter< let stream: MediaStream; - if ( - !this.localUserMediaStream || - (this.localUserMediaStream.getAudioTracks().length === 0 && shouldRequestAudio) || - (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo) || - (this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput) || - (this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) - ) { + let canReuseStream = true; + if (this.localUserMediaStream) { + // This code checks that the device ID is the same as the localUserMediaStream stream, but we update + // the localUserMediaStream whenever the device ID changes (apart from when restoring) so it's not + // clear why this would ever be different, unless there's a race. + if (shouldRequestAudio) { + if ( + this.localUserMediaStream.getAudioTracks().length === 0 || + this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput + ) { + canReuseStream = false; + } + } + if (shouldRequestVideo) { + if ( + this.localUserMediaStream.getVideoTracks().length === 0 || + this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) { + canReuseStream = false; + } + } + } else { + canReuseStream = false; + } + + if (!canReuseStream) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); stream = await navigator.mediaDevices.getUserMedia(constraints); logger.log(`mediaHandler getUserMediaStream streamId ${stream.id} shouldRequestAudio ${