Enable group calls without video and audio track by configuration of MatrixClient (#3162)

* groupCall: add configuration param to allow no audio and no camera

* groupCall: enable datachannel to do no media group calls

* groupCall: changed call no media property as object property

* groupCall: fix existing unit tests

* groupCall: remove not needed flag

* groupCall: rename property to allow no media calls

* groupCall: mute unmute even without device

* groupCall: switch to promise callbacks

* groupCall: switch to try catch

* test: filter dummy code from coverage

* test: extend media mute tests

* groupCall: move permission check to device handler

* mediaHandler: add error in log statement

.gitignore

@@ -19,3 +19,4 @@ out
 
 .vscode
 .vscode/
+.idea/

spec/unit/webrtc/groupCall.spec.ts

@@ -109,7 +109,7 @@ const mockGetStateEvents = (type: EventType, userId?: string): MatrixEvent[] | M
 const ONE_HOUR = 1000 * 60 * 60;
 
 const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise<GroupCall> => {
-    const groupCall = new GroupCall(cli, room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID);
+    const groupCall = new GroupCall(cli, room, GroupCallType.Video, false, GroupCallIntent.Prompt, false, FAKE_CONF_ID);
 
     await groupCall.create();
     await groupCall.enter();
@@ -135,7 +135,7 @@ describe("Group Call", function () {
             mockClient = typedMockClient as unknown as MatrixClient;
 
             room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1);
-            groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt);
+            groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt, false);
             room.currentState.members[FAKE_USER_ID_1] = {
                 userId: FAKE_USER_ID_1,
                 membership: "join",
@@ -484,7 +484,7 @@ describe("Group Call", function () {
         describe("PTT calls", () => {
             beforeEach(async () => {
                 // replace groupcall with a PTT one
-                groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt);
+                groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt, false);
 
                 await groupCall.create();
 
@@ -647,6 +647,7 @@ describe("Group Call", function () {
                 GroupCallType.Video,
                 false,
                 GroupCallIntent.Prompt,
+                false,
                 FAKE_CONF_ID,
             );
 
@@ -656,6 +657,7 @@ describe("Group Call", function () {
                 GroupCallType.Video,
                 false,
                 GroupCallIntent.Prompt,
+                false,
                 FAKE_CONF_ID,
             );
         });
@@ -882,11 +884,27 @@ describe("Group Call", function () {
                 expect(await groupCall.setMicrophoneMuted(false)).toBe(false);
             });
 
+            it("returns false when no permission for audio stream", async () => {
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+                jest.spyOn(mockClient.getMediaHandler(), "getUserMediaStream").mockRejectedValueOnce(
+                    new Error("No Permission"),
+                );
+                expect(await groupCall.setMicrophoneMuted(false)).toBe(false);
+            });
+
             it("returns false when unmuting video with no video device", async () => {
                 const groupCall = await createAndEnterGroupCall(mockClient, room);
                 jest.spyOn(mockClient.getMediaHandler(), "hasVideoDevice").mockResolvedValue(false);
                 expect(await groupCall.setLocalVideoMuted(false)).toBe(false);
             });
+
+            it("returns false when no permission for video stream", async () => {
+                const groupCall = await createAndEnterGroupCall(mockClient, room);
+                jest.spyOn(mockClient.getMediaHandler(), "getUserMediaStream").mockRejectedValueOnce(
+                    new Error("No Permission"),
+                );
+                expect(await groupCall.setLocalVideoMuted(false)).toBe(false);
+            });
         });
 
         describe("remote muting", () => {
@@ -1465,6 +1483,7 @@ describe("Group Call", function () {
                 GroupCallType.Video,
                 false,
                 GroupCallIntent.Prompt,
+                false,
                 FAKE_CONF_ID,
             );
             await groupCall.create();

spec/unit/webrtc/mediaHandler.spec.ts

@@ -242,6 +242,11 @@ describe("Media Handler", function () {
             );
             expect(await mediaHandler.hasAudioDevice()).toEqual(false);
         });
+
+        it("returns false if the system not permitting access audio inputs", async () => {
+            mockMediaDevices.enumerateDevices.mockRejectedValueOnce(new Error("No Permission"));
+            expect(await mediaHandler.hasAudioDevice()).toEqual(false);
+        });
     });
 
     describe("hasVideoDevice", () => {
@@ -255,6 +260,11 @@ describe("Media Handler", function () {
             );
             expect(await mediaHandler.hasVideoDevice()).toEqual(false);
         });
+
+        it("returns false if the system not permitting access video inputs", async () => {
+            mockMediaDevices.enumerateDevices.mockRejectedValueOnce(new Error("No Permission"));
+            expect(await mediaHandler.hasVideoDevice()).toEqual(false);
+        });
     });
 
     describe("getUserMediaStream", () => {

src/client.ts

@@ -371,6 +371,13 @@ export interface ICreateClientOpts {
      * Defaults to a built-in English handler with basic pluralisation.
      */
     roomNameGenerator?: (roomId: string, state: RoomNameState) => string | null;
+
+    /**
+     * If true, participant can join group call without video and audio this has to be allowed. By default, a local
+     * media stream is needed to establish a group call.
+     * Default: false.
+     */
+    isVoipWithNoMediaAllowed?: boolean;
 }
 
 export interface IMatrixClientCreateOpts extends ICreateClientOpts {
@@ -1169,6 +1176,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
     public iceCandidatePoolSize = 0; // XXX: Intended private, used in code.
     public idBaseUrl?: string;
     public baseUrl: string;
+    public readonly isVoipWithNoMediaAllowed;
 
     // Note: these are all `protected` to let downstream consumers make mistakes if they want to.
     // We don't technically support this usage, but have reasons to do this.
@@ -1313,6 +1321,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
         this.iceCandidatePoolSize = opts.iceCandidatePoolSize === undefined ? 0 : opts.iceCandidatePoolSize;
         this.supportsCallTransfer = opts.supportsCallTransfer || false;
         this.fallbackICEServerAllowed = opts.fallbackICEServerAllowed || false;
+        this.isVoipWithNoMediaAllowed = opts.isVoipWithNoMediaAllowed || false;
 
         if (opts.useE2eForGroupCall !== undefined) this.useE2eForGroupCall = opts.useE2eForGroupCall;
 
@@ -1880,14 +1889,17 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             throw new Error(`Cannot find room ${roomId}`);
         }
 
+        // Because without Media section a WebRTC connection is not possible, so need a RTCDataChannel to set up a
+        // no media WebRTC connection anyway.
         return new GroupCall(
             this,
             room,
             type,
             isPtt,
             intent,
+            this.isVoipWithNoMediaAllowed,
             undefined,
-            dataChannelsEnabled,
+            dataChannelsEnabled || this.isVoipWithNoMediaAllowed,
             dataChannelOptions,
         ).create();
     }

src/webrtc/call.ts

@@ -390,6 +390,9 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
     // Used to keep the timer for the delay before actually stopping our
     // video track after muting (see setLocalVideoMuted)
     private stopVideoTrackTimer?: ReturnType<typeof setTimeout>;
+    // Used to allow connection without Video and Audio. To establish a webrtc connection without media a Data channel is
+    // needed At the moment this property is true if we allow MatrixClient with isVoipWithNoMediaAllowed = true
+    private readonly isOnlyDataChannelAllowed: boolean;
 
     /**
      * Construct a new Matrix Call.
@@ -420,6 +423,8 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
             utils.checkObjectHasKeys(server, ["urls"]);
         }
         this.callId = genCallID();
+        // If the Client provides calls without audio and video we need a datachannel for a webrtc connection
+        this.isOnlyDataChannelAllowed = this.client.isVoipWithNoMediaAllowed;
     }
 
     /**
@@ -944,7 +949,11 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
         // According to previous comments in this file, firefox at some point did not
         // add streams until media started arriving on them. Testing latest firefox
         // (81 at time of writing), this is no longer a problem, so let's do it the correct way.
-        if (!remoteStream || remoteStream.getTracks().length === 0) {
+        //
+        // For example in case of no media webrtc connections like screen share only call we have to allow webrtc
+        // connections without remote media. In this case we always use a data channel. At the moment we allow as well
+        // only data channel as media in the WebRTC connection with this setup here.
+        if (!this.isOnlyDataChannelAllowed && (!remoteStream || remoteStream.getTracks().length === 0)) {
             logger.error(
                 `Call ${this.callId} initWithInvite() no remote stream or no tracks after setting remote description!`,
             );

src/webrtc/groupCall.ts

@@ -216,6 +216,7 @@ export class GroupCall extends TypedEventEmitter<
         public type: GroupCallType,
         public isPtt: boolean,
         public intent: GroupCallIntent,
+        public readonly allowCallWithoutVideoAndAudio: boolean,
         groupCallId?: string,
         private dataChannelsEnabled?: boolean,
         private dataChannelOptions?: IGroupCallDataChannelOptions,
@@ -374,8 +375,15 @@ export class GroupCall extends TypedEventEmitter<
         try {
             stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video);
         } catch (error) {
-            this.state = GroupCallState.LocalCallFeedUninitialized;
-            throw error;
+            // If is allowed to join a call without a media stream, then we
+            // don't throw an error here. But we need an empty Local Feed to establish
+            // a connection later.
+            if (this.allowCallWithoutVideoAndAudio) {
+                stream = new MediaStream();
+            } else {
+                this.state = GroupCallState.LocalCallFeedUninitialized;
+                throw error;
+            }
         }
 
         // The call could've been disposed while we were waiting, and could
@@ -584,6 +592,31 @@ export class GroupCall extends TypedEventEmitter<
             logger.log(
                 `GroupCall ${this.groupCallId} setMicrophoneMuted() (streamId=${this.localCallFeed.stream.id}, muted=${muted})`,
             );
+
+            // We needed this here to avoid an error in case user join a call without a device.
+            // I can not use .then .catch functions because linter :-(
+            try {
+                if (!muted) {
+                    const stream = await this.client
+                        .getMediaHandler()
+                        .getUserMediaStream(true, !this.localCallFeed.isVideoMuted());
+                    if (stream === null) {
+                        // if case permission denied to get a stream stop this here
+                        /* istanbul ignore next */
+                        logger.log(
+                            `GroupCall ${this.groupCallId} setMicrophoneMuted() no device to receive local stream, muted=${muted}`,
+                        );
+                        return false;
+                    }
+                }
+            } catch (e) {
+                /* istanbul ignore next */
+                logger.log(
+                    `GroupCall ${this.groupCallId} setMicrophoneMuted() no device or permission to receive local stream, muted=${muted}`,
+                );
+                return false;
+            }
+
             this.localCallFeed.setAudioVideoMuted(muted, null);
             // I don't believe its actually necessary to enable these tracks: they
             // are the one on the GroupCall's own CallFeed and are cloned before being
@@ -617,14 +650,24 @@ export class GroupCall extends TypedEventEmitter<
         }
 
         if (this.localCallFeed) {
+            /* istanbul ignore next */
             logger.log(
                 `GroupCall ${this.groupCallId} setLocalVideoMuted() (stream=${this.localCallFeed.stream.id}, muted=${muted})`,
             );
 
-            const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted);
-            await this.updateLocalUsermediaStream(stream);
-            this.localCallFeed.setAudioVideoMuted(null, muted);
-            setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted);
+            try {
+                const stream = await this.client.getMediaHandler().getUserMediaStream(true, !muted);
+                await this.updateLocalUsermediaStream(stream);
+                this.localCallFeed.setAudioVideoMuted(null, muted);
+                setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted);
+            } catch (_) {
+                // No permission to video device
+                /* istanbul ignore next */
+                logger.log(
+                    `GroupCall ${this.groupCallId} setLocalVideoMuted() no device or permission to receive local stream, muted=${muted}`,
+                );
+                return false;
+            }
         } else {
             logger.log(`GroupCall ${this.groupCallId} setLocalVideoMuted() no stream muted (muted=${muted})`);
             this.initWithVideoMuted = muted;

src/webrtc/groupCallEventHandler.ts

@@ -183,8 +183,11 @@ export class GroupCallEventHandler {
             callType,
             isPtt,
             callIntent,
+            this.client.isVoipWithNoMediaAllowed,
             groupCallId,
-            content?.dataChannelsEnabled,
+            // Because without Media section a WebRTC connection is not possible, so need a RTCDataChannel to set up a
+            // no media WebRTC connection anyway.
+            content?.dataChannelsEnabled || this.client.isVoipWithNoMediaAllowed,
             dataChannelOptions,
         );
 

src/webrtc/mediaHandler.ts

@@ -185,13 +185,23 @@ export class MediaHandler extends TypedEventEmitter<
     }
 
     public async hasAudioDevice(): Promise<boolean> {
-        const devices = await navigator.mediaDevices.enumerateDevices();
-        return devices.filter((device) => device.kind === "audioinput").length > 0;
+        try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            return devices.filter((device) => device.kind === "audioinput").length > 0;
+        } catch (err) {
+            logger.log(`MediaHandler hasAudioDevice() calling navigator.mediaDevices.enumerateDevices with error`, err);
+            return false;
+        }
     }
 
     public async hasVideoDevice(): Promise<boolean> {
-        const devices = await navigator.mediaDevices.enumerateDevices();
-        return devices.filter((device) => device.kind === "videoinput").length > 0;
+        try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            return devices.filter((device) => device.kind === "videoinput").length > 0;
+        } catch (err) {
+            logger.log(`MediaHandler hasVideoDevice() calling navigator.mediaDevices.enumerateDevices with error`, err);
+            return false;
+        }
     }
 
     /**