From ba8577f2685bac834516bf4123a75147a70c1864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 7 Mar 2021 14:15:48 +0100 Subject: [PATCH 01/29] Add CallFeed class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/webrtc/callFeed.ts diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts new file mode 100644 index 000000000..0585ccae3 --- /dev/null +++ b/src/webrtc/callFeed.ts @@ -0,0 +1,28 @@ +/* +Copyright 2021 Šimon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export enum CallFeedType { + Webcam = "webcam", + Screenshare = "screenshare", +} + +export class CallFeed { + constructor( + public stream: MediaStream, + public userId: string, + public type: CallFeedType, + ) {} +} From 530b60cbc26d888e68b8e2ddbc089dbe95d84d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 7 Mar 2021 14:17:21 +0100 Subject: [PATCH 02/29] Make MatrixCall use CallFeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 193 +++++++++-------------------------------- src/webrtc/callFeed.ts | 29 ++++++- 2 files changed, 70 insertions(+), 152 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a9c8799cd..b07d9d832 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -29,6 +29,7 @@ import {EventType} from '../@types/event'; import { RoomMember } from '../models/room-member'; import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, MCallOfferNegotiate, CallCapabilities } from './callEventTypes'; +import { CallFeed, CallFeedType } from './callFeed'; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -101,6 +102,8 @@ export enum CallEvent { RemoteHoldUnhold = 'remote_hold_unhold', // backwards compat alias for LocalHoldUnhold: remove in a major version bump HoldUnhold = 'hold_unhold', + // Feeds have changed + FeedsChanged = 'feeds_changed', } export enum CallErrorCode { @@ -248,8 +251,7 @@ export class MatrixCall extends EventEmitter { private candidateSendTries: number; private sentEndOfCandidates: boolean; private peerConn: RTCPeerConnection; - private localVideoElement: HTMLVideoElement; - private remoteVideoElement: HTMLVideoElement; + private feeds: Array; private remoteAudioElement: HTMLAudioElement; private screenSharingStream: MediaStream; private remoteStream: MediaStream; @@ -327,6 +329,8 @@ export class MatrixCall extends EventEmitter { this.unholdingRemote = false; this.micMuted = false; this.vidMuted = false; + + this.feeds = []; } /** @@ -343,17 +347,11 @@ export class MatrixCall extends EventEmitter { /** * Place a video call to this room. - * @param {Element} remoteVideoElement a <video> DOM element - * to render video to. - * @param {Element} localVideoElement a <video> DOM element - * to render the local camera preview. * @throws If you have not specified a listener for 'error' events. */ - async placeVideoCall(remoteVideoElement: HTMLVideoElement, localVideoElement: HTMLVideoElement) { + async placeVideoCall() { logger.debug("placeVideoCall"); this.checkForErrorListener(); - this.localVideoElement = localVideoElement; - this.remoteVideoElement = remoteVideoElement; const constraints = getUserMediaContraints(ConstraintsType.Video); await this.placeCallWithConstraints(constraints); this.type = CallType.Video; @@ -363,22 +361,11 @@ export class MatrixCall extends EventEmitter { * Place a screen-sharing call to this room. This includes audio. * This method is EXPERIMENTAL and subject to change without warning. It * only works in Google Chrome and Firefox >= 44. - * @param {Element} remoteVideoElement a <video> DOM element - * to render video to. - * @param {Element} localVideoElement a <video> DOM element - * to render the local camera preview. * @throws If you have not specified a listener for 'error' events. */ - async placeScreenSharingCall( - remoteVideoElement: HTMLVideoElement, - localVideoElement: HTMLVideoElement, - selectDesktopCapturerSource: () => Promise, - ) { + async placeScreenSharingCall(selectDesktopCapturerSource: () => Promise) { logger.debug("placeScreenSharingCall"); this.checkForErrorListener(); - this.localVideoElement = localVideoElement; - this.remoteVideoElement = remoteVideoElement; - try { const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); if (!screenshareConstraints) return; @@ -414,21 +401,29 @@ export class MatrixCall extends EventEmitter { return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); } - /** - * Retrieve the local <video> DOM element. - * @return {Element} The dom element - */ - public getLocalVideoElement(): HTMLVideoElement { - return this.localVideoElement; + public getFeeds(): Array { + return this.feeds; } - /** - * Retrieve the remote <video> DOM element - * used for playing back video capable streams. - * @return {Element} The dom element - */ - public getRemoteVideoElement(): HTMLVideoElement { - return this.remoteVideoElement; + public noIncomingFeeds(): boolean { + return !this.feeds.some((feed) => !feed.isLocal()); + } + + private pushNewFeed(stream: MediaStream, userId: string, type: CallFeedType) { + // Try to find a feed with the same stream id as the new stream, + // if we find it replace the old stream with the new one + const feed = this.feeds.find((feed) => feed.stream.id === stream.id); + if (feed) { + feed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed(stream, userId, type, this.client)); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + } + + private deleteAllFeeds() { + this.feeds = []; + this.emit(CallEvent.FeedsChanged, this.feeds); } /** @@ -440,48 +435,6 @@ export class MatrixCall extends EventEmitter { return this.remoteAudioElement; } - /** - * Set the local <video> DOM element. If this call is active, - * video will be rendered to it immediately. - * @param {Element} element The <video> DOM element. - */ - public async setLocalVideoElement(element : HTMLVideoElement) { - this.localVideoElement = element; - - if (element && this.localAVStream && this.type === CallType.Video) { - element.autoplay = true; - - element.srcObject = this.localAVStream; - element.muted = true; - try { - await element.play(); - } catch (e) { - logger.info("Failed to play local video element", e); - } - } - } - - /** - * Set the remote <video> DOM element. If this call is active, - * the first received video-capable stream will be rendered to it immediately. - * @param {Element} element The <video> DOM element. - */ - public setRemoteVideoElement(element : HTMLVideoElement) { - if (element === this.remoteVideoElement) return; - - element.autoplay = true; - - // if we already have an audio element set, use that instead and mute the audio - // on this video element. - if (this.remoteAudioElement) element.muted = true; - - this.remoteVideoElement = element; - - if (this.remoteStream) { - this.playRemoteVideo(); - } - } - /** * Set the remote <audio> DOM element. If this call is active, * the first received audio-only stream will be rendered to it immediately. @@ -641,8 +594,6 @@ export class MatrixCall extends EventEmitter { newCall.gotUserMediaForAnswer(this.localAVStream); delete(this.localAVStream); } - newCall.localVideoElement = this.localVideoElement; - newCall.remoteVideoElement = this.remoteVideoElement; newCall.remoteAudioElement = this.remoteAudioElement; this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -814,8 +765,6 @@ export class MatrixCall extends EventEmitter { if (this.remoteOnHold) { if (this.remoteAudioElement && this.remoteAudioElement.srcObject === this.remoteStream) { this.remoteAudioElement.muted = true; - } else if (this.remoteVideoElement && this.remoteVideoElement.srcObject === this.remoteStream) { - this.remoteVideoElement.muted = true; } } else { this.playRemoteAudio(); @@ -842,24 +791,13 @@ export class MatrixCall extends EventEmitter { logger.debug("gotUserMediaForInvite -> " + this.type); - const videoEl = this.getLocalVideoElement(); - - if (videoEl && this.type === CallType.Video) { - videoEl.autoplay = true; - if (this.screenSharingStream) { - logger.debug( - "Setting screen sharing stream to the local video element", - ); - videoEl.srcObject = this.screenSharingStream; - } else { - videoEl.srcObject = stream; - } - videoEl.muted = true; - try { - await videoEl.play(); - } catch (e) { - logger.info("Failed to play local video element", e); - } + if (this.screenSharingStream) { + logger.debug( + "Setting screen sharing stream to the local video element", + ); + this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), CallFeedType.Screenshare); + } else { + this.pushNewFeed(stream, this.client.getUserId(), CallFeedType.Webcam); } // why do we enable audio (and only audio) tracks here? -- matthew @@ -929,19 +867,7 @@ export class MatrixCall extends EventEmitter { return; } - const localVidEl = this.getLocalVideoElement(); - - if (localVidEl && this.type === CallType.Video) { - localVidEl.autoplay = true; - localVidEl.srcObject = stream; - - localVidEl.muted = true; - try { - await localVidEl.play(); - } catch (e) { - logger.info("Failed to play local video element", e); - } - } + this.pushNewFeed(stream, this.client.getUserId(), CallFeedType.Webcam); this.localAVStream = stream; logger.info("Got local AV stream with id " + this.localAVStream.id); @@ -1361,13 +1287,10 @@ export class MatrixCall extends EventEmitter { logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - if (ev.track.kind === 'video') { - if (this.remoteVideoElement) { - this.playRemoteVideo(); - } - } else { - if (this.remoteAudioElement) this.playRemoteAudio(); - } + this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, CallFeedType.Webcam) + if (this.remoteAudioElement) this.playRemoteAudio(); + + logger.info("playing remote. stream active? " + this.remoteStream.active); }; onNegotiationNeeded = async () => { @@ -1391,9 +1314,7 @@ export class MatrixCall extends EventEmitter { }; async playRemoteAudio() { - if (this.remoteVideoElement) this.remoteVideoElement.muted = true; this.remoteAudioElement.muted = false; - this.remoteAudioElement.srcObject = this.remoteStream; // if audioOutput is non-default: @@ -1417,25 +1338,6 @@ export class MatrixCall extends EventEmitter { } } - private async playRemoteVideo() { - // A note on calling methods on media elements: - // We used to have queues per media element to serialise all calls on those elements. - // The reason given for this was that load() and play() were racing. However, we now - // never call load() explicitly so this seems unnecessary. However, serialising every - // operation was causing bugs where video would not resume because some play command - // had got stuck and all media operations were queued up behind it. If necessary, we - // should serialise the ones that need to be serialised but then be able to interrupt - // them with another load() which will cancel the pending one, but since we don't call - // load() explicitly, it shouldn't be a problem. - this.remoteVideoElement.srcObject = this.remoteStream; - logger.info("playing remote video. stream active? " + this.remoteStream.active); - try { - await this.remoteVideoElement.play(); - } catch (e) { - logger.info("Failed to play remote video element", e); - } - } - onHangupReceived = (msg) => { logger.debug("Hangup received for call ID " + this.callId); @@ -1552,14 +1454,7 @@ export class MatrixCall extends EventEmitter { this.inviteTimeout = null; } - const remoteVid = this.getRemoteVideoElement(); const remoteAud = this.getRemoteAudioElement(); - const localVid = this.getLocalVideoElement(); - - if (remoteVid) { - remoteVid.pause(); - remoteVid.srcObject = null; - } if (remoteAud) { remoteAud.pause(); remoteAud.srcObject = null; @@ -1571,10 +1466,8 @@ export class MatrixCall extends EventEmitter { logger.warn("Failed to set sink ID back to default"); } } - if (localVid) { - localVid.pause(); - localVid.srcObject = null; - } + this.deleteAllFeeds(); + this.hangupParty = hangupParty; this.hangupReason = hangupReason; this.setState(CallState.Ended); diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 0585ccae3..e65466dab 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -14,15 +14,40 @@ See the License for the specific language governing permissions and limitations under the License. */ +import EventEmitter from "events"; + export enum CallFeedType { Webcam = "webcam", Screenshare = "screenshare", } -export class CallFeed { +export enum CallFeedEvent { + NewStream = "new_stream", +} + +export class CallFeed extends EventEmitter { constructor( public stream: MediaStream, public userId: string, public type: CallFeedType, - ) {} + private client: any, // Fix when client is TSified + ) { + super() + } + + public isLocal() { + return this.userId === this.client.getUserId(); + } + + // TODO: This should be later replaced by a method + // that will also check if the remote is muted. + public isAudioOnly(): boolean { + // We assume only one video track + return !this.stream.getTracks().some((track) => track.kind === "video"); + } + + public setNewStream(newStream: MediaStream) { + this.stream = newStream; + this.emit(CallFeedEvent.NewStream, this.stream); + } } From 059430bd0acaba419ac88a256ba23e696b8edb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 10 Mar 2021 09:13:21 +0100 Subject: [PATCH 03/29] Doc public methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 9 +++++++++ src/webrtc/callFeed.ts | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b07d9d832..25f8e8914 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -401,10 +401,19 @@ export class MatrixCall extends EventEmitter { return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); } + /** + * Returns an array of all CallFeeds + * @returns {Array} CallFeeds + */ public getFeeds(): Array { return this.feeds; } + /** + * Returns true if there are no incoming feeds, + * otherwise returns false + * @returns {boolean} no incoming feeds + */ public noIncomingFeeds(): boolean { return !this.feeds.some((feed) => !feed.isLocal()); } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index e65466dab..7ac29a384 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -35,17 +35,30 @@ export class CallFeed extends EventEmitter { super() } + /** + * Returns true if CallFeed is local, otherwise returns false + * @returns {boolean} is local? + */ public isLocal() { return this.userId === this.client.getUserId(); } // TODO: This should be later replaced by a method // that will also check if the remote is muted. + /** + * Returns true if there are no video tracks, otherwise returns false + * @returns {boolean} is audio only? + */ public isAudioOnly(): boolean { // We assume only one video track return !this.stream.getTracks().some((track) => track.kind === "video"); } + /** + * Replaces the current MediaStream with a new one. + * This method should be only used by MatrixCall. + * @param newStream new stream with which to replace the current one + */ public setNewStream(newStream: MediaStream) { this.stream = newStream; this.emit(CallFeedEvent.NewStream, this.stream); From b307a177f40d38a234490e7db09884ec4592bfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 9 Mar 2021 08:03:36 +0100 Subject: [PATCH 04/29] Remove handling of audio from MatrixCall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/matrix.ts | 1 - src/webrtc/call.ts | 83 ---------------------------------------------- 2 files changed, 84 deletions(-) diff --git a/src/matrix.ts b/src/matrix.ts index f1014c5de..144cd07c0 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -55,7 +55,6 @@ export * from "./content-repo"; export * as ContentHelpers from "./content-helpers"; export { createNewMatrixCall, - setAudioOutput as setMatrixCallAudioOutput, setAudioInput as setMatrixCallAudioInput, setVideoInput as setMatrixCallVideoInput, } from "./webrtc/call"; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 25f8e8914..1950088b4 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -252,7 +252,6 @@ export class MatrixCall extends EventEmitter { private sentEndOfCandidates: boolean; private peerConn: RTCPeerConnection; private feeds: Array; - private remoteAudioElement: HTMLAudioElement; private screenSharingStream: MediaStream; private remoteStream: MediaStream; private localAVStream: MediaStream; @@ -435,29 +434,6 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - /** - * Retrieve the remote <audio> DOM element - * used for playing back audio only streams. - * @return {Element} The dom element - */ - public getRemoteAudioElement(): HTMLAudioElement { - return this.remoteAudioElement; - } - - /** - * Set the remote <audio> DOM element. If this call is active, - * the first received audio-only stream will be rendered to it immediately. - * The audio will *not* be rendered from the remoteVideoElement. - * @param {Element} element The <video> DOM element. - */ - public async setRemoteAudioElement(element: HTMLAudioElement) { - if (element === this.remoteAudioElement) return; - - this.remoteAudioElement = element; - - if (this.remoteStream) this.playRemoteAudio(); - } - // The typescript definitions have this type as 'any' :( public async getCurrentCallStats(): Promise { if (this.callHasEnded()) { @@ -603,7 +579,6 @@ export class MatrixCall extends EventEmitter { newCall.gotUserMediaForAnswer(this.localAVStream); delete(this.localAVStream); } - newCall.remoteAudioElement = this.remoteAudioElement; this.successor = newCall; this.emit(CallEvent.Replaced, newCall); this.hangup(CallErrorCode.Replaced, true); @@ -713,10 +688,6 @@ export class MatrixCall extends EventEmitter { } this.updateMuteStatus(); - if (!onHold) { - this.playRemoteAudio(); - } - this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold); } @@ -770,14 +741,6 @@ export class MatrixCall extends EventEmitter { const vidShouldBeMuted = this.vidMuted || this.remoteOnHold; setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted); - - if (this.remoteOnHold) { - if (this.remoteAudioElement && this.remoteAudioElement.srcObject === this.remoteStream) { - this.remoteAudioElement.muted = true; - } - } else { - this.playRemoteAudio(); - } } /** @@ -1297,7 +1260,6 @@ export class MatrixCall extends EventEmitter { logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, CallFeedType.Webcam) - if (this.remoteAudioElement) this.playRemoteAudio(); logger.info("playing remote. stream active? " + this.remoteStream.active); }; @@ -1322,31 +1284,6 @@ export class MatrixCall extends EventEmitter { } }; - async playRemoteAudio() { - this.remoteAudioElement.muted = false; - this.remoteAudioElement.srcObject = this.remoteStream; - - // if audioOutput is non-default: - try { - if (audioOutput) { - // This seems quite unreliable in Chrome, although I haven't yet managed to make a jsfiddle where - // it fails. - // It seems reliable if you set the sink ID after setting the srcObject and then set the sink ID - // back to the default after the call is over - logger.info("Setting audio sink to " + audioOutput + ", was " + this.remoteAudioElement.sinkId); - await this.remoteAudioElement.setSinkId(audioOutput); - } - } catch (e) { - logger.warn("Couldn't set requested audio output device: using default", e); - } - - try { - await this.remoteAudioElement.play(); - } catch (e) { - logger.error("Failed to play remote audio element", e); - } - } - onHangupReceived = (msg) => { logger.debug("Hangup received for call ID " + this.callId); @@ -1463,18 +1400,6 @@ export class MatrixCall extends EventEmitter { this.inviteTimeout = null; } - const remoteAud = this.getRemoteAudioElement(); - if (remoteAud) { - remoteAud.pause(); - remoteAud.srcObject = null; - try { - // As per comment in playRemoteAudio, setting the sink ID back to the default - // once the call is over makes setSinkId work reliably. - await this.remoteAudioElement.setSinkId('') - } catch (e) { - logger.warn("Failed to set sink ID back to default"); - } - } this.deleteAllFeeds(); this.hangupParty = hangupParty; @@ -1730,16 +1655,8 @@ async function getScreenshareContraints(selectDesktopCapturerSource?: () => Prom } } -let audioOutput: string; let audioInput: string; let videoInput: string; -/** - * Set an audio output device to use for MatrixCalls - * @function - * @param {string=} deviceId the identifier for the device - * undefined treated as unset - */ -export function setAudioOutput(deviceId: string) { audioOutput = deviceId; } /** * Set an audio input device to use for MatrixCalls * @function From e78b41583273ca6b458fd165b38a0d5b4cabc6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 4 Apr 2021 08:33:51 +0200 Subject: [PATCH 05/29] Add getMember() to CallFeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- src/webrtc/callFeed.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3529a17ee..f3aa3b571 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -424,7 +424,7 @@ export class MatrixCall extends EventEmitter { if (feed) { feed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, type, this.client)); + this.feeds.push(new CallFeed(stream, userId, type, this.client, this.roomId)); this.emit(CallEvent.FeedsChanged, this.feeds); } } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 7ac29a384..959889a96 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -31,10 +31,20 @@ export class CallFeed extends EventEmitter { public userId: string, public type: CallFeedType, private client: any, // Fix when client is TSified + private roomId: string, ) { super() } + /** + * Returns callRoom member + * @returns member of the callRoom + */ + public getMember() { + const callRoom = this.client.getRoom(this.roomId); + return callRoom.getMember(this.userId); + } + /** * Returns true if CallFeed is local, otherwise returns false * @returns {boolean} is local? From cdc0d5623b405995edb5f9a40dbe0e7547fa8796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 4 Apr 2021 08:37:09 +0200 Subject: [PATCH 06/29] Rename to match MSC3077 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 14 +++++++------- src/webrtc/callFeed.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f3aa3b571..bb73bb858 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -29,7 +29,7 @@ import {EventType} from '../@types/event'; import { RoomMember } from '../models/room-member'; import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, MCallOfferNegotiate, CallCapabilities } from './callEventTypes'; -import { CallFeed, CallFeedType } from './callFeed'; +import { CallFeed, CallFeedPurpose } from './callFeed'; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -417,14 +417,14 @@ export class MatrixCall extends EventEmitter { return !this.feeds.some((feed) => !feed.isLocal()); } - private pushNewFeed(stream: MediaStream, userId: string, type: CallFeedType) { + private pushNewFeed(stream: MediaStream, userId: string, purpose: CallFeedPurpose) { // Try to find a feed with the same stream id as the new stream, // if we find it replace the old stream with the new one const feed = this.feeds.find((feed) => feed.stream.id === stream.id); if (feed) { feed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, type, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); this.emit(CallEvent.FeedsChanged, this.feeds); } } @@ -770,9 +770,9 @@ export class MatrixCall extends EventEmitter { logger.debug( "Setting screen sharing stream to the local video element", ); - this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), CallFeedType.Screenshare); + this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), CallFeedPurpose.Screenshare); } else { - this.pushNewFeed(stream, this.client.getUserId(), CallFeedType.Webcam); + this.pushNewFeed(stream, this.client.getUserId(), CallFeedPurpose.Usermedia); } // why do we enable audio (and only audio) tracks here? -- matthew @@ -842,7 +842,7 @@ export class MatrixCall extends EventEmitter { return; } - this.pushNewFeed(stream, this.client.getUserId(), CallFeedType.Webcam); + this.pushNewFeed(stream, this.client.getUserId(), CallFeedPurpose.Usermedia); this.localAVStream = stream; logger.info("Got local AV stream with id " + this.localAVStream.id); @@ -1266,7 +1266,7 @@ export class MatrixCall extends EventEmitter { logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, CallFeedType.Webcam) + this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, CallFeedPurpose.Usermedia) logger.info("playing remote. stream active? " + this.remoteStream.active); }; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 959889a96..78b37c7b4 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -16,8 +16,8 @@ limitations under the License. import EventEmitter from "events"; -export enum CallFeedType { - Webcam = "webcam", +export enum CallFeedPurpose { + Usermedia = "usermedia", Screenshare = "screenshare", } @@ -29,7 +29,7 @@ export class CallFeed extends EventEmitter { constructor( public stream: MediaStream, public userId: string, - public type: CallFeedType, + public purpose: CallFeedPurpose, private client: any, // Fix when client is TSified private roomId: string, ) { From 32830b93f12e9c37de2b3b0c511603f41ca1fc28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 4 Apr 2021 08:50:27 +0200 Subject: [PATCH 07/29] Rename audioOnly to videoMuted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes more sense and will match a possible mute events MSC Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 78b37c7b4..1a934f0fa 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -53,13 +53,23 @@ export class CallFeed extends EventEmitter { return this.userId === this.client.getUserId(); } - // TODO: This should be later replaced by a method - // that will also check if the remote is muted. + // TODO: The two following methods should be later replaced + // by something that will also check if the remote is muted /** - * Returns true if there are no video tracks, otherwise returns false - * @returns {boolean} is audio only? + * Returns true if audio is muted or if there are no audio + * tracks, otherwise returns false + * @returns {boolean} is audio muted? */ - public isAudioOnly(): boolean { + public isAudioMuted(): boolean { + return !this.stream.getTracks().some((track) => track.kind === "audio"); + } + + /** + * Returns true video is muted or if there are no video + * tracks, otherwise returns false + * @returns {boolean} is video muted? + */ + public isVideoMuted(): boolean { // We assume only one video track return !this.stream.getTracks().some((track) => track.kind === "video"); } From 7f4397f8ca334a07a54b3cc8e313886100c9c4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 5 Apr 2021 10:31:27 +0200 Subject: [PATCH 08/29] Use getAudioTracks() and getVideoTracks() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is much nicer. Before I hadn't realized this was possible Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 1a934f0fa..de4c88570 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -61,7 +61,7 @@ export class CallFeed extends EventEmitter { * @returns {boolean} is audio muted? */ public isAudioMuted(): boolean { - return !this.stream.getTracks().some((track) => track.kind === "audio"); + return this.stream.getAudioTracks().length === 0; } /** @@ -71,7 +71,7 @@ export class CallFeed extends EventEmitter { */ public isVideoMuted(): boolean { // We assume only one video track - return !this.stream.getTracks().some((track) => track.kind === "video"); + return this.stream.getVideoTracks().length === 0; } /** From 2916a73f4c8e2506887822481d9f4c3d1ff241a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Apr 2021 11:01:17 +0200 Subject: [PATCH 09/29] Make selectDesktopCapturerSource optional --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1a270c5ca..16e1fca1e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -362,7 +362,7 @@ export class MatrixCall extends EventEmitter { * only works in Google Chrome and Firefox >= 44. * @throws If you have not specified a listener for 'error' events. */ - async placeScreenSharingCall(selectDesktopCapturerSource: () => Promise) { + async placeScreenSharingCall(selectDesktopCapturerSource?: () => Promise) { logger.debug("placeScreenSharingCall"); this.checkForErrorListener(); try { From ee828d454a8253fb4872b8993e0d4e323c0d6696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Apr 2021 11:07:22 +0200 Subject: [PATCH 10/29] Rename CallFeedPurpose to SDPStreamMetadataPurpose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is to match MSC3077 Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 12 ++++++------ src/webrtc/callFeed.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1a270c5ca..c34061176 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -29,7 +29,7 @@ import {EventType} from '../@types/event'; import { RoomMember } from '../models/room-member'; import { randomString } from '../randomstring'; import { MCallReplacesEvent, MCallAnswer, MCallOfferNegotiate, CallCapabilities } from './callEventTypes'; -import { CallFeed, CallFeedPurpose } from './callFeed'; +import { CallFeed, SDPStreamMetadataPurpose } from './callFeed'; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -422,7 +422,7 @@ export class MatrixCall extends EventEmitter { return !this.feeds.some((feed) => !feed.isLocal()); } - private pushNewFeed(stream: MediaStream, userId: string, purpose: CallFeedPurpose) { + private pushNewFeed(stream: MediaStream, userId: string, purpose: SDPStreamMetadataPurpose) { // Try to find a feed with the same stream id as the new stream, // if we find it replace the old stream with the new one const feed = this.feeds.find((feed) => feed.stream.id === stream.id); @@ -775,9 +775,9 @@ export class MatrixCall extends EventEmitter { logger.debug( "Setting screen sharing stream to the local video element", ); - this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), CallFeedPurpose.Screenshare); + this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), SDPStreamMetadataPurpose.Screenshare); } else { - this.pushNewFeed(stream, this.client.getUserId(), CallFeedPurpose.Usermedia); + this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); } // why do we enable audio (and only audio) tracks here? -- matthew @@ -847,7 +847,7 @@ export class MatrixCall extends EventEmitter { return; } - this.pushNewFeed(stream, this.client.getUserId(), CallFeedPurpose.Usermedia); + this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); this.localAVStream = stream; logger.info("Got local AV stream with id " + this.localAVStream.id); @@ -1271,7 +1271,7 @@ export class MatrixCall extends EventEmitter { logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, CallFeedPurpose.Usermedia) + this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) logger.info("playing remote. stream active? " + this.remoteStream.active); }; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index de4c88570..e24f0a2f6 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -16,7 +16,7 @@ limitations under the License. import EventEmitter from "events"; -export enum CallFeedPurpose { +export enum SDPStreamMetadataPurpose { Usermedia = "usermedia", Screenshare = "screenshare", } @@ -29,7 +29,7 @@ export class CallFeed extends EventEmitter { constructor( public stream: MediaStream, public userId: string, - public purpose: CallFeedPurpose, + public purpose: SDPStreamMetadataPurpose, private client: any, // Fix when client is TSified private roomId: string, ) { From ddc5bd3b36cb2efaee3083079560459467f39acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Apr 2021 11:07:53 +0200 Subject: [PATCH 11/29] Add prefixes to SDPStreamMetadataPurpose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index e24f0a2f6..617e04a8c 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -17,8 +17,8 @@ limitations under the License. import EventEmitter from "events"; export enum SDPStreamMetadataPurpose { - Usermedia = "usermedia", - Screenshare = "screenshare", + Usermedia = "m.usermedia", + Screenshare = "m.screenshare", } export enum CallFeedEvent { From 1f2b3512c1b45cb018e880f3ee04c23c1d06149e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 8 Apr 2021 11:14:42 +0200 Subject: [PATCH 12/29] Move SDPStreamMetadataPurpose into callEventTypes.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 11 +++++++++-- src/webrtc/callEventTypes.ts | 5 +++++ src/webrtc/callFeed.ts | 6 +----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c34061176..c920f2954 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -28,8 +28,15 @@ import MatrixEvent from '../models/event'; import {EventType} from '../@types/event'; import { RoomMember } from '../models/room-member'; import { randomString } from '../randomstring'; -import { MCallReplacesEvent, MCallAnswer, MCallOfferNegotiate, CallCapabilities } from './callEventTypes'; -import { CallFeed, SDPStreamMetadataPurpose } from './callFeed'; +import { + MCallReplacesEvent, + MCallAnswer, + MCallOfferNegotiate, + CallCapabilities, + SDPStreamMetadataPurpose, +} from './callEventTypes'; +import { CallFeed } from './callFeed'; + // events: hangup, error(err), replaced(call), state(state, oldState) diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index c68903bb8..dce146485 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -1,6 +1,11 @@ // allow camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ +export enum SDPStreamMetadataPurpose { + Usermedia = "m.usermedia", + Screenshare = "m.screenshare", +} + interface CallOfferAnswer { type: string; sdp: string; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 617e04a8c..c5ad2af49 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -15,11 +15,7 @@ limitations under the License. */ import EventEmitter from "events"; - -export enum SDPStreamMetadataPurpose { - Usermedia = "m.usermedia", - Screenshare = "m.screenshare", -} +import {SDPStreamMetadataPurpose} from "./callEventTypes"; export enum CallFeedEvent { NewStream = "new_stream", From 1bfaa28f9c9944cb00888f7378cdabad2363e345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Apr 2021 11:09:28 +0200 Subject: [PATCH 13/29] Fix missing types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index c5ad2af49..e71c1d725 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -16,6 +16,8 @@ limitations under the License. import EventEmitter from "events"; import {SDPStreamMetadataPurpose} from "./callEventTypes"; +import MatrixClient from "../client" +import {RoomMember} from "../models/room-member"; export enum CallFeedEvent { NewStream = "new_stream", @@ -26,7 +28,7 @@ export class CallFeed extends EventEmitter { public stream: MediaStream, public userId: string, public purpose: SDPStreamMetadataPurpose, - private client: any, // Fix when client is TSified + private client: MatrixClient, // Fix when client is TSified private roomId: string, ) { super() @@ -36,7 +38,7 @@ export class CallFeed extends EventEmitter { * Returns callRoom member * @returns member of the callRoom */ - public getMember() { + public getMember(): RoomMember { const callRoom = this.client.getRoom(this.roomId); return callRoom.getMember(this.userId); } @@ -45,7 +47,7 @@ export class CallFeed extends EventEmitter { * Returns true if CallFeed is local, otherwise returns false * @returns {boolean} is local? */ - public isLocal() { + public isLocal(): boolean { return this.userId === this.client.getUserId(); } From 72a0931663bd260a9b9a2d11a4891a7c1caec930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 16 Apr 2021 11:38:01 +0200 Subject: [PATCH 14/29] Remove comment: // Fix when client is TSified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index e71c1d725..b0004639f 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -28,7 +28,7 @@ export class CallFeed extends EventEmitter { public stream: MediaStream, public userId: string, public purpose: SDPStreamMetadataPurpose, - private client: MatrixClient, // Fix when client is TSified + private client: MatrixClient, private roomId: string, ) { super() From 4667f8be0387eab57a1ae5d47e33789b3cd44891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 24 Apr 2021 12:08:45 +0200 Subject: [PATCH 15/29] Use feeds in stopAllMedia() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7917b49ef..94805bdd5 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1478,19 +1478,9 @@ export class MatrixCall extends EventEmitter { private stopAllMedia() { logger.debug(`stopAllMedia (stream=${this.localAVStream})`); - if (this.localAVStream) { - for (const track of this.localAVStream.getTracks()) { - track.stop(); - } - } - if (this.screenSharingStream) { - for (const track of this.screenSharingStream.getTracks()) { - track.stop(); - } - } - if (this.remoteStream) { - for (const track of this.remoteStream.getTracks()) { + for (const feed of this.feeds) { + for (const track of feed.stream.getTracks()) { track.stop(); } } From 0e2e906b24b85036d12d35635e48070f900bb151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 24 Apr 2021 12:24:15 +0200 Subject: [PATCH 16/29] Remove remoteStream prop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is done in order to be more generic Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 94805bdd5..58d06969c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -260,7 +260,6 @@ export class MatrixCall extends EventEmitter { private peerConn: RTCPeerConnection; private feeds: Array; private screenSharingStream: MediaStream; - private remoteStream: MediaStream; private localAVStream: MediaStream; private inviteOrAnswerSent: boolean; private waitForLocalAVStream: boolean; @@ -497,17 +496,19 @@ export class MatrixCall extends EventEmitter { this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } + + let remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; // According to previous comments in this file, firefox at some point did not // add streams until media started ariving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. - if (!this.remoteStream || this.remoteStream.getTracks().length === 0) { + if (!remoteStream || remoteStream.getTracks().length === 0) { logger.error("No remote stream or no tracks after setting remote description!"); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } - this.type = this.remoteStream.getTracks().some(t => t.kind === 'video') ? CallType.Video : CallType.Voice; + this.type = remoteStream.getTracks().some(t => t.kind === 'video') ? CallType.Video : CallType.Voice; this.setState(CallState.Ringing); @@ -1259,28 +1260,31 @@ export class MatrixCall extends EventEmitter { logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); return; } + + let remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; + // If we already have a stream, check this track is from the same one - if (this.remoteStream && ev.streams[0].id !== this.remoteStream.id) { + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when tranciever directionality is changed and the 'active' + // status of streams change - Dave + if (remoteStream && ev.streams[0].id !== remoteStream.id) { logger.warn( - `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${this.remoteStream.id}`, + `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${remoteStream.id}`, ); return; } - if (!this.remoteStream) { + if (!remoteStream) { logger.info("Got remote stream with id " + ev.streams[0].id); } - // Note that we check by ID above and always set the remote stream: Chrome appears - // to make new stream objects when tranciever directionality is changed and the 'active' - // status of streams change - this.remoteStream = ev.streams[0]; + remoteStream = ev.streams[0]; logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - this.pushNewFeed(this.remoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) + this.pushNewFeed(remoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) - logger.info("playing remote. stream active? " + this.remoteStream.active); + logger.info("playing remote. stream active? " + remoteStream.active); }; onNegotiationNeeded = async () => { From 1fe92f10c15db36946c4dc84108789eae7590fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 24 Apr 2021 12:26:54 +0200 Subject: [PATCH 17/29] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 58d06969c..247aa0134 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -496,8 +496,8 @@ export class MatrixCall extends EventEmitter { this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } - - let remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; + + const remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; // According to previous comments in this file, firefox at some point did not // add streams until media started ariving on them. Testing latest firefox From 973de2db558e055983d9f1df180449f967670a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 24 Apr 2021 12:56:28 +0200 Subject: [PATCH 18/29] stopAllMedia() before deleteAllFeeds() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 247aa0134..12cd0e99b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1466,12 +1466,13 @@ export class MatrixCall extends EventEmitter { this.inviteTimeout = null; } + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() + this.stopAllMedia(); this.deleteAllFeeds(); this.hangupParty = hangupParty; this.hangupReason = hangupReason; this.setState(CallState.Ended); - this.stopAllMedia(); if (this.peerConn && this.peerConn.signalingState !== 'closed') { this.peerConn.close(); } From 88b310c394b08037d6315d041029f3f5c83a4cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 26 Apr 2021 16:00:17 +0200 Subject: [PATCH 19/29] Rename stuff to make it easy to read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 12cd0e99b..6602bc840 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1261,30 +1261,30 @@ export class MatrixCall extends EventEmitter { return; } - let remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; + const oldRemoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; // If we already have a stream, check this track is from the same one // Note that we check by ID and always set the remote stream: Chrome appears // to make new stream objects when tranciever directionality is changed and the 'active' // status of streams change - Dave - if (remoteStream && ev.streams[0].id !== remoteStream.id) { + if (oldRemoteStream && ev.streams[0].id !== oldRemoteStream.id) { logger.warn( - `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${remoteStream.id}`, + `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${oldRemoteStream.id}`, ); return; } - if (!remoteStream) { + if (!oldRemoteStream) { logger.info("Got remote stream with id " + ev.streams[0].id); } - remoteStream = ev.streams[0]; + const newRemoteStream = ev.streams[0]; logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - this.pushNewFeed(remoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) + this.pushNewFeed(newRemoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) - logger.info("playing remote. stream active? " + remoteStream.active); + logger.info("playing remote. stream active? " + newRemoteStream.active); }; onNegotiationNeeded = async () => { From e11c523a750366c33e814b59a400364926021d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 27 Apr 2021 10:06:21 +0200 Subject: [PATCH 20/29] Add getLocalFeeds() and getRemoteFeeds() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6602bc840..c966e5738 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -419,6 +419,22 @@ export class MatrixCall extends EventEmitter { return this.feeds; } + /** + * Returns an array of all local CallFeeds + * @returns {Array} local CallFeeds + */ + public getLocalFeeds(): Array { + return this.feeds.filter((feed) => {return feed.isLocal()}); + } + + /** + * Returns an array of all remote CallFeeds + * @returns {Array} remote CallFeeds + */ + public getRemoteFeeds(): Array { + return this.feeds.filter((feed) => {return !feed.isLocal()}); + } + /** * Returns true if there are no incoming feeds, * otherwise returns false From ad80d693691200847ac0d6a26d59a86a6afac050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 30 Apr 2021 14:17:30 +0200 Subject: [PATCH 21/29] Add some basic styling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- examples/voip/index.html | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/examples/voip/index.html b/examples/voip/index.html index a3259cfa1..ab96790a6 100644 --- a/examples/voip/index.html +++ b/examples/voip/index.html @@ -1,26 +1,34 @@ + -VoIP Test - - + VoIP Test + + + - You can place and receive calls with this example. Make sure to edit the + You can place and receive calls with this example. Make sure to edit the constants in browserTest.js first.
-
-
- -
-
-
-
- -
+
+ +
+ + + \ No newline at end of file From 4b3c8b29698a8546c2cf04aa5c3bfe8cab7df1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 30 Apr 2021 14:18:20 +0200 Subject: [PATCH 22/29] Update the example to work with the new feed code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- examples/voip/browserTest.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/examples/voip/browserTest.js b/examples/voip/browserTest.js index 01f463421..f4a3751ae 100644 --- a/examples/voip/browserTest.js +++ b/examples/voip/browserTest.js @@ -31,6 +31,23 @@ function addListeners(call) { call.hangup(); disableButtons(false, true, true); }); + call.on("feeds_changed", function(feeds) { + const localFeed = feeds.find((feed) => feed.isLocal()); + const remoteFeed = feeds.find((feed) => !feed.isLocal()); + + const remoteElement = document.getElementById("remote"); + const localElement = document.getElementById("local"); + + if (remoteFeed && remoteFeed.stream) { + remoteElement.srcObject = remoteFeed.stream; + remoteElement.play(); + } + if (localFeed && localFeed.stream) { + localElement.muted = true; + localElement.srcObject = localFeed.stream; + localElement.play(); + } + }); } window.onload = function() { @@ -62,10 +79,7 @@ function syncComplete() { ); console.log("Call => %s", call); addListeners(call); - call.placeVideoCall( - document.getElementById("remote"), - document.getElementById("local") - ); + call.placeVideoCall(); document.getElementById("result").innerHTML = "

Placed call.

"; disableButtons(true, true, false); }; From 3b7d6f8334006706d8710741893a69bfd15a9f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 30 Apr 2021 14:43:00 +0200 Subject: [PATCH 23/29] This check doesn't seem to be necessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- examples/voip/browserTest.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/voip/browserTest.js b/examples/voip/browserTest.js index f4a3751ae..fbf47c30b 100644 --- a/examples/voip/browserTest.js +++ b/examples/voip/browserTest.js @@ -38,11 +38,11 @@ function addListeners(call) { const remoteElement = document.getElementById("remote"); const localElement = document.getElementById("local"); - if (remoteFeed && remoteFeed.stream) { + if (remoteFeed) { remoteElement.srcObject = remoteFeed.stream; remoteElement.play(); } - if (localFeed && localFeed.stream) { + if (localFeed) { localElement.muted = true; localElement.srcObject = localFeed.stream; localElement.play(); From 62a34848c76f09ed588ce0fe648ac0e4d67f2c66 Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Tue, 4 May 2021 15:37:54 +0100 Subject: [PATCH 24/29] Prepare changelog for v10.1.0-rc.1 --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2eac122a..89a9d0b71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +Changes in [10.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v10.1.0-rc.1) (2021-05-04) +============================================================================================================ +[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v10.0.0...v10.1.0-rc.1) + + * Revert "Raise logging dramatically to chase pending event errors" + [\#1681](https://github.com/matrix-org/matrix-js-sdk/pull/1681) + * Add test coverage collection script + [\#1677](https://github.com/matrix-org/matrix-js-sdk/pull/1677) + * Raise logging dramatically to chase pending event errors + [\#1678](https://github.com/matrix-org/matrix-js-sdk/pull/1678) + * Support MSC3086 asserted identity + [\#1674](https://github.com/matrix-org/matrix-js-sdk/pull/1674) + * Fix `/search` with no results field work again + [\#1670](https://github.com/matrix-org/matrix-js-sdk/pull/1670) + * Add room.getMembers method + [\#1672](https://github.com/matrix-org/matrix-js-sdk/pull/1672) + Changes in [10.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v10.0.0) (2021-04-26) ================================================================================================== [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v10.0.0-rc.1...v10.0.0) From 4e967c979ce81ca547d887a0f22217260fe4e53c Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Tue, 4 May 2021 15:37:55 +0100 Subject: [PATCH 25/29] v10.1.0-rc.1 --- package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index f2e66dc76..84a8c0cae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "10.0.0", + "version": "10.1.0-rc.1", "description": "Matrix Client-Server SDK for Javascript", "scripts": { "prepublishOnly": "yarn build", @@ -28,7 +28,7 @@ "keywords": [ "matrix-org" ], - "main": "./src/index.ts", + "main": "./lib/index.js", "browser": "./lib/browser-index.js", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.js", @@ -108,5 +108,6 @@ "coverageReporters": [ "text" ] - } + }, + "typings": "./lib/index.d.ts" } From 80059174520affbe228f9b2f7eedb8503deb4f58 Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Mon, 10 May 2021 14:43:17 +0100 Subject: [PATCH 26/29] Prepare changelog for v10.1.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89a9d0b71..f5dc8a32f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +Changes in [10.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v10.1.0) (2021-05-10) +================================================================================================== +[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v10.1.0-rc.1...v10.1.0) + + * No changes since rc.1 + Changes in [10.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v10.1.0-rc.1) (2021-05-04) ============================================================================================================ [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v10.0.0...v10.1.0-rc.1) From fda13875efd21db72b61023c942b9e998646ec79 Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Mon, 10 May 2021 14:43:18 +0100 Subject: [PATCH 27/29] v10.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 84a8c0cae..5fdcaaaad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "10.1.0-rc.1", + "version": "10.1.0", "description": "Matrix Client-Server SDK for Javascript", "scripts": { "prepublishOnly": "yarn build", From 7c4ced8f46ab11b7cb1b07723d27b8a76828cb01 Mon Sep 17 00:00:00 2001 From: RiotRobot Date: Mon, 10 May 2021 14:46:14 +0100 Subject: [PATCH 28/29] Resetting package fields for development --- package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 5fdcaaaad..d9ed10c73 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "keywords": [ "matrix-org" ], - "main": "./lib/index.js", + "main": "./src/index.ts", "browser": "./lib/browser-index.js", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.js", @@ -108,6 +108,5 @@ "coverageReporters": [ "text" ] - }, - "typings": "./lib/index.d.ts" + } } From 4ca718adc41b9b09bc140056616b83b15bd4617f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 13:46:58 +0000 Subject: [PATCH 29/29] Bump hosted-git-info from 2.8.8 to 2.8.9 Bumps [hosted-git-info](https://github.com/npm/hosted-git-info) from 2.8.8 to 2.8.9. - [Release notes](https://github.com/npm/hosted-git-info/releases) - [Changelog](https://github.com/npm/hosted-git-info/blob/v2.8.9/CHANGELOG.md) - [Commits](https://github.com/npm/hosted-git-info/compare/v2.8.8...v2.8.9) Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index b002ba624..9e97396d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3522,9 +3522,9 @@ hmac-drbg@^1.0.1: minimalistic-crypto-utils "^1.0.1" hosted-git-info@^2.1.4: - version "2.8.8" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" - integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== + version "2.8.9" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" + integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== html-encoding-sniffer@^2.0.1: version "2.0.1"