diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3ff6f011c..ce62768e4 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,14 +1,15 @@ import EventEmitter from "events"; -import { CallFeed, CallFeedEvent } from "./callFeed"; +import { CallFeed } from "./callFeed"; import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; -import { CallErrorCode, CallEvent, CallState, CallType, MatrixCall } from "./call"; +import { CallErrorCode, CallState, CallType, MatrixCall } from "./call"; import { RoomMember } from "../models/room-member"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { Room } from "../models/room"; import { logger } from "../logger"; import { Callback } from "../client"; import { ReEmitter } from "../ReEmitter"; +import { GroupCallParticipant, GroupCallParticipantEvent } from "./groupCallParticipant"; export enum GroupCallEvent { Entered = "entered", @@ -20,217 +21,6 @@ export enum GroupCallEvent { const CONF_ROOM = "me.robertlong.conf"; const CONF_PARTICIPANT = "me.robertlong.conf.participant"; -const PARTICIPANT_TIMEOUT = 1000 * 15; -const SPEAKING_THRESHOLD = -80; -const ACTIVE_SPEAKER_INTERVAL = 1000; -const ACTIVE_SPEAKER_SAMPLES = 8; - -export enum GroupCallParticipantEvent { - Speaking = "speaking", - VolumeChanged = "volume_changed", - MuteStateChanged = "mute_state_changed", - Datachannel = "datachannel", - CallReplaced = "call_replaced" -} - -export class GroupCallParticipant extends EventEmitter { - public feeds: CallFeed[] = []; - public activeSpeaker: boolean; - public activeSpeakerSamples: number[]; - public dataChannel?: RTCDataChannel; - - constructor( - private groupCall: GroupCall, - public member: RoomMember, - // The session id is used to re-initiate calls if the user's participant - // session id has changed - public sessionId: string, - public call?: MatrixCall, - ) { - super(); - - this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( - -Infinity, - ); - - if (this.call) { - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - } - } - - public replaceCall(call: MatrixCall, sessionId: string) { - const oldCall = this.call; - - if (this.call) { - this.call.hangup(CallErrorCode.Replaced, false); - this.call.removeListener(CallEvent.State, this.onCallStateChanged); - this.call.removeListener( - CallEvent.FeedsChanged, - this.onCallFeedsChanged, - ); - this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); - this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); - } - - this.call = call; - this.member = call.getOpponentMember(); - this.activeSpeaker = false; - this.sessionId = sessionId; - - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.DataChannel, this.onCallDataChannel); - - this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); - } - - public get usermediaFeed() { - return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get usermediaStream(): MediaStream { - return this.usermediaFeed?.stream; - } - - public isAudioMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isAudioMuted(); - } - - public isVideoMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isVideoMuted(); - } - - private onCallStateChanged = (state) => { - const call = this.call; - const audioMuted = this.groupCall.localParticipant.isAudioMuted(); - - if ( - call.localUsermediaStream && - call.isMicrophoneMuted() !== audioMuted - ) { - call.setMicrophoneMuted(audioMuted); - } - - const videoMuted = this.groupCall.localParticipant.isVideoMuted(); - - if ( - call.localUsermediaStream && - call.isLocalVideoMuted() !== videoMuted - ) { - call.setLocalVideoMuted(videoMuted); - } - }; - - onCallFeedsChanged = () => { - const oldFeeds = this.feeds; - const newFeeds = this.call.getRemoteFeeds(); - - this.feeds = []; - - for (const feed of newFeeds) { - if (oldFeeds.includes(feed)) { - continue; - } - - this.addCallFeed(feed); - } - }; - - onCallReplaced = (newCall) => { - // TODO: Should we always reuse the sessionId? - this.replaceCall(newCall, this.sessionId); - }; - - onCallHangup = () => { - if (this.call.hangupReason === CallErrorCode.Replaced) { - return; - } - - const participantIndex = this.groupCall.participants.indexOf(this); - - if (participantIndex === -1) { - return; - } - - this.groupCall.participants.splice(participantIndex, 1); - - if ( - this.groupCall.activeSpeaker === this && - this.groupCall.participants.length > 0 - ) { - this.groupCall.activeSpeaker = this.groupCall.participants[0]; - this.groupCall.activeSpeaker.activeSpeaker = true; - this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); - } - - this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); - }; - - addCallFeed(callFeed: CallFeed) { - if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { - callFeed.setSpeakingThreshold(SPEAKING_THRESHOLD); - callFeed.measureVolumeActivity(true); - callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - callFeed.on( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - callFeed.on( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - this.onCallFeedMuteStateChanged( - this.isAudioMuted(), - this.isVideoMuted(), - ); - } - - this.feeds.push(callFeed); - } - - onCallFeedSpeaking = (speaking: boolean) => { - this.emit(GroupCallParticipantEvent.Speaking, speaking); - }; - - onCallFeedVolumeChanged = (maxVolume: number) => { - this.activeSpeakerSamples.shift(); - this.activeSpeakerSamples.push(maxVolume); - this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); - }; - - onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { - if (audioMuted) { - this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( - -Infinity, - ); - } - - this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); - }; - - onCallDataChannel = (dataChannel: RTCDataChannel) => { - this.dataChannel = dataChannel; - this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); - }; -} export class GroupCall extends EventEmitter { public entered = false; @@ -238,6 +28,10 @@ export class GroupCall extends EventEmitter { public localParticipant: GroupCallParticipant; public participants: GroupCallParticipant[] = []; public room: Room; + public activeSpeakerSampleCount = 8; + public activeSpeakerInterval = 1000; + public speakingThreshold = -80; + public participantTimeout = 1000 * 15; private speakerMap: Map = new Map(); private presenceLoopTimeout?: number; @@ -476,7 +270,7 @@ export class GroupCall extends EventEmitter { ...currentMemberState.getContent(), [CONF_PARTICIPANT]: { sessionId: this.localParticipant.sessionId, - expiresAt: new Date().getTime() + PARTICIPANT_TIMEOUT * 2, + expiresAt: new Date().getTime() + this.participantTimeout * 2, }, }, userId, @@ -513,7 +307,7 @@ export class GroupCall extends EventEmitter { this.presenceLoopTimeout = setTimeout( this.onPresenceLoop, - PARTICIPANT_TIMEOUT, + this.participantTimeout, ); }; @@ -698,10 +492,10 @@ export class GroupCall extends EventEmitter { for (let i = 0; i < participant.activeSpeakerSamples.length; i++) { const volume = participant.activeSpeakerSamples[i]; - total += Math.max(volume, SPEAKING_THRESHOLD); + total += Math.max(volume, this.speakingThreshold); } - const avg = total / ACTIVE_SPEAKER_SAMPLES; + const avg = total / this.activeSpeakerSampleCount; if (!topAvg || avg > topAvg) { topAvg = avg; @@ -709,7 +503,7 @@ export class GroupCall extends EventEmitter { } } - if (nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { + if (nextActiveSpeaker && topAvg > this.speakingThreshold) { if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) { this.activeSpeaker.activeSpeaker = false; nextActiveSpeaker.activeSpeaker = true; @@ -720,7 +514,7 @@ export class GroupCall extends EventEmitter { this.activeSpeakerLoopTimeout = setTimeout( this.onActiveSpeakerLoop, - ACTIVE_SPEAKER_INTERVAL, + this.activeSpeakerInterval, ); }; diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts new file mode 100644 index 000000000..8ab909bc9 --- /dev/null +++ b/src/webrtc/groupCallParticipant.ts @@ -0,0 +1,213 @@ +import EventEmitter from "events"; +import { CallFeed, CallFeedEvent } from "./callFeed"; +import { CallErrorCode, CallEvent, MatrixCall } from "./call"; +import { RoomMember } from "../models/room-member"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { GroupCall, GroupCallEvent } from "./groupCall"; + +export enum GroupCallParticipantEvent { + Speaking = "speaking", + VolumeChanged = "volume_changed", + MuteStateChanged = "mute_state_changed", + Datachannel = "datachannel", + CallReplaced = "call_replaced" +} + +export class GroupCallParticipant extends EventEmitter { + public feeds: CallFeed[] = []; + public activeSpeaker: boolean; + public activeSpeakerSamples: number[]; + public dataChannel?: RTCDataChannel; + + constructor( + private groupCall: GroupCall, + public member: RoomMember, + // The session id is used to re-initiate calls if the user's participant + // session id has changed + public sessionId: string, + public call?: MatrixCall, + ) { + super(); + + this.activeSpeakerSamples = Array(groupCall.activeSpeakerSampleCount).fill( + -Infinity, + ); + + if (this.call) { + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + } + } + + public replaceCall(call: MatrixCall, sessionId: string) { + const oldCall = this.call; + + if (this.call) { + this.call.hangup(CallErrorCode.Replaced, false); + this.call.removeListener(CallEvent.State, this.onCallStateChanged); + this.call.removeListener( + CallEvent.FeedsChanged, + this.onCallFeedsChanged, + ); + this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); + this.call.removeListener(CallEvent.Hangup, this.onCallHangup); + this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); + } + + this.call = call; + this.member = call.getOpponentMember(); + this.activeSpeaker = false; + this.sessionId = sessionId; + + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + this.call.on(CallEvent.DataChannel, this.onCallDataChannel); + + this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); + } + + public get usermediaFeed() { + return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get usermediaStream(): MediaStream { + return this.usermediaFeed?.stream; + } + + public isAudioMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isAudioMuted(); + } + + public isVideoMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isVideoMuted(); + } + + private onCallStateChanged = (state) => { + const call = this.call; + const audioMuted = this.groupCall.localParticipant.isAudioMuted(); + + if ( + call.localUsermediaStream && + call.isMicrophoneMuted() !== audioMuted + ) { + call.setMicrophoneMuted(audioMuted); + } + + const videoMuted = this.groupCall.localParticipant.isVideoMuted(); + + if ( + call.localUsermediaStream && + call.isLocalVideoMuted() !== videoMuted + ) { + call.setLocalVideoMuted(videoMuted); + } + }; + + onCallFeedsChanged = () => { + const oldFeeds = this.feeds; + const newFeeds = this.call.getRemoteFeeds(); + + this.feeds = []; + + for (const feed of newFeeds) { + if (oldFeeds.includes(feed)) { + continue; + } + + this.addCallFeed(feed); + } + }; + + onCallReplaced = (newCall) => { + // TODO: Should we always reuse the sessionId? + this.replaceCall(newCall, this.sessionId); + }; + + onCallHangup = () => { + if (this.call.hangupReason === CallErrorCode.Replaced) { + return; + } + + const participantIndex = this.groupCall.participants.indexOf(this); + + if (participantIndex === -1) { + return; + } + + this.groupCall.participants.splice(participantIndex, 1); + + if ( + this.groupCall.activeSpeaker === this && + this.groupCall.participants.length > 0 + ) { + this.groupCall.activeSpeaker = this.groupCall.participants[0]; + this.groupCall.activeSpeaker.activeSpeaker = true; + this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); + } + + this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); + }; + + addCallFeed(callFeed: CallFeed) { + if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { + callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); + callFeed.measureVolumeActivity(true); + callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + callFeed.on( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + callFeed.on( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + this.onCallFeedMuteStateChanged( + this.isAudioMuted(), + this.isVideoMuted(), + ); + } + + this.feeds.push(callFeed); + } + + onCallFeedSpeaking = (speaking: boolean) => { + this.emit(GroupCallParticipantEvent.Speaking, speaking); + }; + + onCallFeedVolumeChanged = (maxVolume: number) => { + this.activeSpeakerSamples.shift(); + this.activeSpeakerSamples.push(maxVolume); + this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); + }; + + onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { + if (audioMuted) { + this.activeSpeakerSamples = Array(this.groupCall.activeSpeakerSampleCount).fill( + -Infinity, + ); + } + + this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); + }; + + onCallDataChannel = (dataChannel: RTCDataChannel) => { + this.dataChannel = dataChannel; + this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); + }; +}