diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7afa8f08a..c443232e4 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -416,10 +416,26 @@ export class MatrixCall extends EventEmitter { return this.localUsermediaFeed?.stream; } - private get localScreensharingStream(): MediaStream { + public get localScreensharingStream(): MediaStream { return this.localScreensharingFeed?.stream; } + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + private getFeedByStreamId(streamId: string): CallFeed { return this.getFeeds().find((feed) => feed.stream.id === streamId); } @@ -551,51 +567,87 @@ export class MatrixCall extends EventEmitter { logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); } - private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { const userId = this.client.getUserId(); + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + // We try to replace an existing feed if there already is one with the same purpose const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: stream.getAudioTracks().length === 0, - videoMuted: stream.getVideoTracks().length === 0, - userId, - stream, - purpose, - })); + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); this.emit(CallEvent.FeedsChanged, this.feeds); } + } - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + this.feeds.push(callFeed); if (addToPeerConnection) { - const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ? + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? this.usermediaSenders : this.screensharingSenders; // Empty the array senderArray.splice(0, senderArray.length); - this.emit(CallEvent.FeedsChanged, this.feeds); - for (const track of stream.getTracks()) { + for (const track of callFeed.stream.getTracks()) { logger.info( `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${purpose}"` + + `streamId="${callFeed.stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + `) to peer connection`, ); - senderArray.push(this.peerConn.addTrack(track, stream)); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); } } - logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`); + logger.info( + `Pushed local stream ` + + `(id="${callFeed.stream.id}", ` + + `active="${callFeed.stream.active}", ` + + `purpose="${callFeed.purpose}")`, + ); + + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeedByStream(callFeed.stream); } private deleteAllFeeds(): void { @@ -760,11 +812,27 @@ export class MatrixCall extends EventEmitter { this.waitForLocalAVStream = true; try { - const mediaStream = await this.client.getMediaHandler().getUserMediaStream( + const stream = await this.client.getMediaHandler().getUserMediaStream( answerWithAudio, answerWithVideo, ); this.waitForLocalAVStream = false; - this.gotUserMediaForAnswer(mediaStream); + const usermediaFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); } catch (e) { if (answerWithVideo) { // Try to answer without video @@ -782,6 +850,14 @@ export class MatrixCall extends EventEmitter { } } + public answerWithCallFeeds(callFeeds: CallFeed[]): void { + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + this.gotCallFeedsForAnswer(callFeeds); + } + /** * Replace this call with a new call, e.g. for glare resolution. Used by * MatrixClient. @@ -793,7 +869,7 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - newCall.gotUserMediaForAnswer(this.localUsermediaStream); + newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -865,7 +941,7 @@ export class MatrixCall extends EventEmitter { if (this.hasLocalUserMediaAudioTrack) return; if (this.hasLocalUserMediaVideoTrack) return; - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } else if (upgradeAudio) { if (this.hasLocalUserMediaAudioTrack) return; @@ -931,7 +1007,7 @@ export class MatrixCall extends EventEmitter { try { const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); if (!stream) return false; - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { this.emit(CallEvent.Error, @@ -973,7 +1049,7 @@ export class MatrixCall extends EventEmitter { }); sender.replaceTrack(track); - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { @@ -1133,13 +1209,9 @@ export class MatrixCall extends EventEmitter { setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); } - /** - * Internal - * @param {Object} stream - */ - private gotUserMediaForInvite = async (stream: MediaStream): Promise => { + private gotCallFeedsForInvite(callFeeds: CallFeed[]): void { if (this.successor) { - this.successor.gotUserMediaForAnswer(stream); + this.successor.gotCallFeedsForAnswer(callFeeds); return; } if (this.callHasEnded()) { @@ -1147,12 +1219,15 @@ export class MatrixCall extends EventEmitter { return; } - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + this.setState(CallState.CreateOffer); logger.debug("gotUserMediaForInvite"); // Now we wait for the negotiationneeded event - }; + } private async sendAnswer(): Promise { const answerContent = { @@ -1201,12 +1276,15 @@ export class MatrixCall extends EventEmitter { this.sendCandidateQueue(); } - private gotUserMediaForAnswer = async (stream: MediaStream): Promise => { - if (this.callHasEnded()) { - return; + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + for (const feed of callFeeds) { + this.pushLocalFeed(feed); } - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.setState(CallState.CreateAnswer); let myAnswer; @@ -1234,7 +1312,7 @@ export class MatrixCall extends EventEmitter { this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } - }; + } /** * Internal @@ -1976,15 +2054,41 @@ export class MatrixCall extends EventEmitter { * @throws if have passed audio=false. */ public async placeCall(audio: boolean, video: boolean): Promise { - logger.debug(`placeCall audio=${audio} video=${video}`); if (!audio) { throw new Error("You CANNOT start a call without audio"); } + this.setState(CallState.WaitLocalMedia); + + try { + const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + const callFeed = new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.client.getUserId(), + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, + }); + await this.placeCallWithCallFeeds([callFeed]); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + + /** + * Place a call to this room with call feed. + * @param {CallFeed[]} callFeeds to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise { this.checkForErrorListener(); + this.direction = CallDirection.Outbound; + // XXX Find a better way to do this this.client.callEventHandler.calls.set(this.callId, this); - this.setState(CallState.WaitLocalMedia); - this.direction = CallDirection.Outbound; // make sure we have valid turn creds. Unless something's gone wrong, it should // poll and keep the credentials valid so this should be instant. @@ -1996,14 +2100,7 @@ export class MatrixCall extends EventEmitter { // create the peer connection now so it can be gathering candidates while we get user // media (assuming a candidate pool size is configured) this.peerConn = this.createPeerConnection(); - - try { - const mediaStream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - this.gotUserMediaForInvite(mediaStream); - } catch (e) { - this.getUserMediaFailed(e); - return; - } + this.gotCallFeedsForInvite(callFeeds); } private createPeerConnection(): RTCPeerConnection { diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 30a1481bd..d2c4ca80f 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -19,8 +19,9 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; -const POLLING_INTERVAL = 250; // ms -const SPEAKING_THRESHOLD = -60; // dB +const POLLING_INTERVAL = 200; // ms +export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; // samples export interface ICallFeedOpts { client: MatrixClient; @@ -43,6 +44,7 @@ export class CallFeed extends EventEmitter { public stream: MediaStream; public userId: string; public purpose: SDPStreamMetadataPurpose; + public speakingVolumeSamples: number[]; private client: MatrixClient; private roomId: string; @@ -65,6 +67,7 @@ export class CallFeed extends EventEmitter { this.purpose = opts.purpose; this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.updateStream(null, opts.stream); @@ -78,6 +81,8 @@ export class CallFeed extends EventEmitter { } private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) return; + if (oldStream) { oldStream.removeEventListener("addtrack", this.onAddTrack); this.measureVolumeActivity(false); @@ -152,6 +157,10 @@ export class CallFeed extends EventEmitter { return this.stream.getVideoTracks().length === 0 || this.videoMuted; } + public isSpeaking(): boolean { + return this.speaking; + } + /** * Replaces the current MediaStream with a new one. * This method should be only used by MatrixCall. @@ -167,6 +176,7 @@ export class CallFeed extends EventEmitter { */ public setAudioMuted(muted: boolean): void { this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } @@ -191,6 +201,7 @@ export class CallFeed extends EventEmitter { this.volumeLooper(); } else { this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -199,31 +210,43 @@ export class CallFeed extends EventEmitter { this.speakingThreshold = threshold; } - private volumeLooper(): void { + private volumeLooper = () => { if (!this.analyser) return; - this.volumeLooperTimeout = setTimeout(() => { - if (!this.measuringVolumeActivity) return; + if (!this.measuringVolumeActivity) return; - this.analyser.getFloatFrequencyData(this.frequencyBinCount); + this.analyser.getFloatFrequencyData(this.frequencyBinCount); - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; } + } - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - const newSpeaking = maxVolume > this.speakingThreshold; - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; } + } - this.volumeLooper(); - }, POLLING_INTERVAL); - } + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; public dispose(): void { clearTimeout(this.volumeLooperTimeout);