Merge pull request #1911 from SimonBrandner/feature/media-handling

Improve `MatrixCall` media handling and internally remove `CallType`s

src/client.ts

@@ -144,6 +144,7 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/
 import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules";
 import { IThreepid } from "./@types/threepids";
 import { CryptoStore } from "./crypto/store/base";
+import { MediaHandler } from "./webrtc/mediaHandler";
 
 export type Store = IStore;
 export type SessionStore = WebStorageSessionStore;
@@ -733,6 +734,7 @@ export class MatrixClient extends EventEmitter {
     protected checkTurnServersIntervalID: number;
     protected exportedOlmDeviceToImport: IOlmDevice;
     protected txnCtr = 0;
+    protected mediaHandler = new MediaHandler();
 
     constructor(opts: IMatrixClientCreateOpts) {
         super();
@@ -1240,6 +1242,13 @@ export class MatrixClient extends EventEmitter {
         return this.canSupportVoip;
     }
 
+    /**
+     * @returns {MediaHandler}
+     */
+    public getMediaHandler(): MediaHandler {
+        return this.mediaHandler;
+    }
+
     /**
      * Set whether VoIP calls are forced to use only TURN
      * candidates. This is the same as the forceTURN option

src/matrix.ts

@@ -49,8 +49,6 @@ export * from "./content-repo";
 export * as ContentHelpers from "./content-helpers";
 export {
     createNewMatrixCall,
-    setAudioInput as setMatrixCallAudioInput,
-    setVideoInput as setMatrixCallVideoInput,
 } from "./webrtc/call";
 
 // expose the underlying request object so different environments can use

src/webrtc/call.ts

@@ -212,11 +212,6 @@ export enum CallErrorCode {
     Transfered = 'transferred',
 }
 
-enum ConstraintsType {
-    Audio = "audio",
-    Video = "video",
-}
-
 /**
  * The version field that we set in m.call.* events
  */
@@ -256,7 +251,6 @@ function genCallID(): string {
  */
 export class MatrixCall extends EventEmitter {
     public roomId: string;
-    public type: CallType = null;
     public callId: string;
     public state = CallState.Fledgling;
     public hangupParty: CallParty;
@@ -337,11 +331,7 @@ export class MatrixCall extends EventEmitter {
      * @throws If you have not specified a listener for 'error' events.
      */
     public async placeVoiceCall(): Promise<void> {
-        logger.debug("placeVoiceCall");
+        await this.placeCall(true, false);
-        this.checkForErrorListener();
-        const constraints = getUserMediaContraints(ConstraintsType.Audio);
-        this.type = CallType.Voice;
-        await this.placeCallWithConstraints(constraints);
     }
 
     /**
@@ -349,11 +339,7 @@ export class MatrixCall extends EventEmitter {
      * @throws If you have not specified a listener for 'error' events.
      */
     public async placeVideoCall(): Promise<void> {
-        logger.debug("placeVideoCall");
+        await this.placeCall(true, true);
-        this.checkForErrorListener();
-        const constraints = getUserMediaContraints(ConstraintsType.Video);
-        this.type = CallType.Video;
-        await this.placeCallWithConstraints(constraints);
     }
 
     public getOpponentMember(): RoomMember {
@@ -372,6 +358,25 @@ export class MatrixCall extends EventEmitter {
         return this.remoteAssertedIdentity;
     }
 
+    public get type(): CallType {
+        return (this.hasLocalUserMediaVideoTrack || this.hasRemoteUserMediaVideoTrack)
+            ? CallType.Video
+            : CallType.Voice;
+    }
+
+    public get hasLocalUserMediaVideoTrack(): boolean {
+        return this.localUsermediaStream?.getVideoTracks().length > 0;
+    }
+
+    public get hasRemoteUserMediaVideoTrack(): boolean {
+        return this.getRemoteFeeds().some((feed) => {
+            return (
+                feed.purpose === SDPStreamMetadataPurpose.Usermedia &&
+                feed.stream.getVideoTracks().length > 0
+            );
+        });
+    }
+
     public get localUsermediaFeed(): CallFeed {
         return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
     }
@@ -635,8 +640,6 @@ export class MatrixCall extends EventEmitter {
             return;
         }
 
-        this.type = remoteStream.getTracks().some(t => t.kind === 'video') ? CallType.Video : CallType.Voice;
-
         this.setState(CallState.Ringing);
 
         if (event.getLocalAge()) {
@@ -674,20 +677,17 @@ export class MatrixCall extends EventEmitter {
             return;
         }
 
-        logger.debug(`Answering call ${this.callId} of type ${this.type}`);
+        logger.debug(`Answering call ${this.callId}`);
 
         if (!this.localUsermediaStream && !this.waitForLocalAVStream) {
-            const constraints = getUserMediaContraints(
-                this.type == CallType.Video ?
-                    ConstraintsType.Video:
-                    ConstraintsType.Audio,
-            );
-            logger.log("Getting user media with constraints", constraints);
             this.setState(CallState.WaitLocalMedia);
             this.waitForLocalAVStream = true;
 
             try {
-                const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+                const mediaStream = await this.client.getMediaHandler().getUserMediaStream(
+                    true,
+                    this.hasRemoteUserMediaVideoTrack,
+                );
                 this.waitForLocalAVStream = false;
                 this.gotUserMediaForAnswer(mediaStream);
             } catch (e) {
@@ -802,7 +802,7 @@ export class MatrixCall extends EventEmitter {
         logger.debug(`Set screensharing enabled? ${enabled}`);
         if (enabled) {
             try {
-                const stream = await getScreensharingStream(desktopCapturerSourceId);
+                const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                 if (!stream) return false;
                 this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
                 return true;
@@ -837,7 +837,7 @@ export class MatrixCall extends EventEmitter {
         logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`);
         if (enabled) {
             try {
-                const stream = await getScreensharingStream(desktopCapturerSourceId);
+                const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId);
                 if (!stream) return false;
 
                 const track = stream.getTracks().find((track) => {
@@ -1007,7 +1007,7 @@ export class MatrixCall extends EventEmitter {
         this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
         this.setState(CallState.CreateOffer);
 
-        logger.debug("gotUserMediaForInvite -> " + this.type);
+        logger.debug("gotUserMediaForInvite");
         // Now we wait for the negotiationneeded event
     };
 
@@ -1814,8 +1814,17 @@ export class MatrixCall extends EventEmitter {
         }
     }
 
-    private async placeCallWithConstraints(constraints: MediaStreamConstraints): Promise<void> {
+    /**
-        logger.log("Getting user media with constraints", constraints);
+     * Place a call to this room.
+     * @throws if you have not specified a listener for 'error' events.
+     * @throws if have passed audio=false.
+     */
+    public async placeCall(audio: boolean, video: boolean): Promise<void> {
+        logger.debug(`placeCall audio=${audio} video=${video}`);
+        if (!audio) {
+            throw new Error("You CANNOT start a call without audio");
+        }
+        this.checkForErrorListener();
         // XXX Find a better way to do this
         this.client.callEventHandler.calls.set(this.callId, this);
         this.setState(CallState.WaitLocalMedia);
@@ -1833,7 +1842,7 @@ export class MatrixCall extends EventEmitter {
         this.peerConn = this.createPeerConnection();
 
         try {
-            const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+            const mediaStream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
             this.gotUserMediaForInvite(mediaStream);
         } catch (e) {
             this.getUserMediaFailed(e);
@@ -1927,96 +1936,12 @@ export class MatrixCall extends EventEmitter {
     }
 }
 
-async function getScreensharingStream(desktopCapturerSourceId: string): Promise<MediaStream> {
-    const screenshareConstraints = getScreenshareContraints(desktopCapturerSourceId);
-    if (!screenshareConstraints) return null;
-
-    if (desktopCapturerSourceId) {
-        // We are using Electron
-        logger.debug("Getting screen stream using getUserMedia()...");
-        return await navigator.mediaDevices.getUserMedia(screenshareConstraints);
-    } else {
-        // We are not using Electron
-        logger.debug("Getting screen stream using getDisplayMedia()...");
-        return await navigator.mediaDevices.getDisplayMedia(screenshareConstraints);
-    }
-}
-
 function setTracksEnabled(tracks: Array<MediaStreamTrack>, enabled: boolean): void {
     for (let i = 0; i < tracks.length; i++) {
         tracks[i].enabled = enabled;
     }
 }
 
-function getUserMediaContraints(type: ConstraintsType): MediaStreamConstraints {
-    const isWebkit = !!navigator.webkitGetUserMedia;
-
-    switch (type) {
-        case ConstraintsType.Audio: {
-            return {
-                audio: {
-                    deviceId: audioInput ? { ideal: audioInput } : undefined,
-                },
-                video: false,
-            };
-        }
-        case ConstraintsType.Video: {
-            return {
-                audio: {
-                    deviceId: audioInput ? { ideal: audioInput } : undefined,
-                }, video: {
-                    deviceId: videoInput ? { ideal: videoInput } : undefined,
-                    /* We want 640x360.  Chrome will give it only if we ask exactly,
-                       FF refuses entirely if we ask exactly, so have to ask for ideal
-                       instead
-                       XXX: Is this still true?
-                     */
-                    width: isWebkit ? { exact: 640 } : { ideal: 640 },
-                    height: isWebkit ? { exact: 360 } : { ideal: 360 },
-                },
-            };
-        }
-    }
-}
-
-function getScreenshareContraints(desktopCapturerSourceId?: string): DesktopCapturerConstraints {
-    if (desktopCapturerSourceId) {
-        logger.debug("Using desktop capturer source", desktopCapturerSourceId);
-        return {
-            audio: false,
-            video: {
-                mandatory: {
-                    chromeMediaSource: "desktop",
-                    chromeMediaSourceId: desktopCapturerSourceId,
-                },
-            },
-        };
-    } else {
-        logger.debug("Not using desktop capturer source");
-        return {
-            audio: false,
-            video: true,
-        };
-    }
-}
-
-let audioInput: string;
-let videoInput: string;
-/**
- * Set an audio input device to use for MatrixCalls
- * @function
- * @param {string=} deviceId the identifier for the device
- * undefined treated as unset
- */
-export function setAudioInput(deviceId: string): void { audioInput = deviceId; }
-/**
- * Set a video input device to use for MatrixCalls
- * @function
- * @param {string=} deviceId the identifier for the device
- * undefined treated as unset
- */
-export function setVideoInput(deviceId: string): void { videoInput = deviceId; }
-
 /**
  * DEPRECATED
  * Use client.createCall()

src/webrtc/mediaHandler.ts

@@ -0,0 +1,115 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+Copyright 2021 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "../logger";
+
+export class MediaHandler {
+    private audioInput: string;
+    private videoInput: string;
+
+    /**
+     * Set an audio input device to use for MatrixCalls
+     * @param {string} deviceId the identifier for the device
+     * undefined treated as unset
+     */
+    public setAudioInput(deviceId: string): void {
+        this.audioInput = deviceId;
+    }
+
+    /**
+     * Set a video input device to use for MatrixCalls
+     * @param {string} deviceId the identifier for the device
+     * undefined treated as unset
+     */
+    public setVideoInput(deviceId: string): void {
+        this.videoInput = deviceId;
+    }
+
+    /**
+     * @returns {MediaStream} based on passed parameters
+     */
+    public async getUserMediaStream(audio: boolean, video: boolean): Promise<MediaStream> {
+        const constraints = this.getUserMediaContraints(audio, video);
+        logger.log("Getting user media with constraints", constraints);
+        return await navigator.mediaDevices.getUserMedia(constraints);
+    }
+
+    /**
+     * @returns {MediaStream} based on passed parameters
+     */
+    public async getScreensharingStream(desktopCapturerSourceId: string): Promise<MediaStream> {
+        const screenshareConstraints = this.getScreenshareContraints(desktopCapturerSourceId);
+        if (!screenshareConstraints) return null;
+
+        if (desktopCapturerSourceId) {
+            // We are using Electron
+            logger.debug("Getting screen stream using getUserMedia()...");
+            return await navigator.mediaDevices.getUserMedia(screenshareConstraints);
+        } else {
+            // We are not using Electron
+            logger.debug("Getting screen stream using getDisplayMedia()...");
+            return await navigator.mediaDevices.getDisplayMedia(screenshareConstraints);
+        }
+    }
+
+    private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints {
+        const isWebkit = !!navigator.webkitGetUserMedia;
+
+        return {
+            audio: audio
+                ? {
+                    deviceId: this.audioInput ? { ideal: this.audioInput } : undefined,
+                }
+                : false,
+            video: video
+                ? {
+                    deviceId: this.videoInput ? { ideal: this.videoInput } : undefined,
+                    /* We want 640x360.  Chrome will give it only if we ask exactly,
+                   FF refuses entirely if we ask exactly, so have to ask for ideal
+                   instead
+                   XXX: Is this still true?
+                 */
+                    width: isWebkit ? { exact: 640 } : { ideal: 640 },
+                    height: isWebkit ? { exact: 360 } : { ideal: 360 },
+                }
+                : false,
+        };
+    }
+
+    private getScreenshareContraints(desktopCapturerSourceId?: string): DesktopCapturerConstraints {
+        if (desktopCapturerSourceId) {
+            logger.debug("Using desktop capturer source", desktopCapturerSourceId);
+            return {
+                audio: false,
+                video: {
+                    mandatory: {
+                        chromeMediaSource: "desktop",
+                        chromeMediaSourceId: desktopCapturerSourceId,
+                    },
+                },
+            };
+        } else {
+            logger.debug("Not using desktop capturer source");
+            return {
+                audio: false,
+                video: true,
+            };
+        }
+    }
+}