Placed call.
"; disableButtons(true, true, false); }; diff --git a/examples/voip/index.html b/examples/voip/index.html index a3259cfa1..ab96790a6 100644 --- a/examples/voip/index.html +++ b/examples/voip/index.html @@ -1,26 +1,34 @@ + -browserTest.js first.
- <video> DOM element
- * to render video to.
- * @param {Element} localVideoElement a <video> DOM element
- * to render the local camera preview.
* @throws If you have not specified a listener for 'error' events.
*/
- async placeVideoCall(remoteVideoElement: HTMLVideoElement, localVideoElement: HTMLVideoElement) {
+ async placeVideoCall() {
logger.debug("placeVideoCall");
this.checkForErrorListener();
- this.localVideoElement = localVideoElement;
- this.remoteVideoElement = remoteVideoElement;
const constraints = getUserMediaContraints(ConstraintsType.Video);
this.type = CallType.Video;
await this.placeCallWithConstraints(constraints);
@@ -372,22 +375,11 @@ export class MatrixCall extends EventEmitter {
* Place a screen-sharing call to this room. This includes audio.
* This method is EXPERIMENTAL and subject to change without warning. It
* only works in Google Chrome and Firefox >= 44.
- * @param {Element} remoteVideoElement a <video> DOM element
- * to render video to.
- * @param {Element} localVideoElement a <video> DOM element
- * to render the local camera preview.
* @throws If you have not specified a listener for 'error' events.
*/
- async placeScreenSharingCall(
- remoteVideoElement: HTMLVideoElement,
- localVideoElement: HTMLVideoElement,
- selectDesktopCapturerSource?: () => Promise<video> DOM element.
- * @return {Element} The dom element
+ * Returns an array of all CallFeeds
+ * @returns {Array<video> DOM element
- * used for playing back video capable streams.
- * @return {Element} The dom element
+ * Returns an array of all local CallFeeds
+ * @returns {Array<audio> DOM element
- * used for playing back audio only streams.
- * @return {Element} The dom element
+ * Returns an array of all remote CallFeeds
+ * @returns {Array<video> DOM element. If this call is active,
- * video will be rendered to it immediately.
- * @param {Element} element The <video> DOM element.
+ * Returns true if there are no incoming feeds,
+ * otherwise returns false
+ * @returns {boolean} no incoming feeds
*/
- public async setLocalVideoElement(element: HTMLVideoElement) {
- this.localVideoElement = element;
+ public noIncomingFeeds(): boolean {
+ return !this.feeds.some((feed) => !feed.isLocal());
+ }
- if (element && this.localAVStream && this.type === CallType.Video) {
- element.autoplay = true;
-
- element.srcObject = this.localAVStream;
- element.muted = true;
- try {
- await element.play();
- } catch (e) {
- logger.info("Failed to play local video element", e);
- }
+ private pushNewFeed(stream: MediaStream, userId: string, purpose: SDPStreamMetadataPurpose) {
+ // Try to find a feed with the same stream id as the new stream,
+ // if we find it replace the old stream with the new one
+ const feed = this.feeds.find((feed) => feed.stream.id === stream.id);
+ if (feed) {
+ feed.setNewStream(stream);
+ } else {
+ this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId));
+ this.emit(CallEvent.FeedsChanged, this.feeds);
}
}
- /**
- * Set the remote <video> DOM element. If this call is active,
- * the first received video-capable stream will be rendered to it immediately.
- * @param {Element} element The <video> DOM element.
- */
- public setRemoteVideoElement(element: HTMLVideoElement) {
- if (element === this.remoteVideoElement) return;
-
- element.autoplay = true;
-
- // if we already have an audio element set, use that instead and mute the audio
- // on this video element.
- if (this.remoteAudioElement) element.muted = true;
-
- this.remoteVideoElement = element;
-
- if (this.remoteStream) {
- this.playRemoteVideo();
- }
- }
-
- /**
- * Set the remote <audio> DOM element. If this call is active,
- * the first received audio-only stream will be rendered to it immediately.
- * The audio will *not* be rendered from the remoteVideoElement.
- * @param {Element} element The <video> DOM element.
- */
- public async setRemoteAudioElement(element: HTMLAudioElement) {
- if (element === this.remoteAudioElement) return;
-
- this.remoteAudioElement = element;
-
- if (this.remoteStream) this.playRemoteAudio();
+ private deleteAllFeeds() {
+ this.feeds = [];
+ this.emit(CallEvent.FeedsChanged, this.feeds);
}
// The typescript definitions have this type as 'any' :(
@@ -566,16 +526,18 @@ export class MatrixCall extends EventEmitter {
return;
}
+ const remoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream;
+
// According to previous comments in this file, firefox at some point did not
// add streams until media started ariving on them. Testing latest firefox
// (81 at time of writing), this is no longer a problem, so let's do it the correct way.
- if (!this.remoteStream || this.remoteStream.getTracks().length === 0) {
+ if (!remoteStream || remoteStream.getTracks().length === 0) {
logger.error("No remote stream or no tracks after setting remote description!");
this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
return;
}
- this.type = this.remoteStream.getTracks().some(t => t.kind === 'video') ? CallType.Video : CallType.Voice;
+ this.type = remoteStream.getTracks().some(t => t.kind === 'video') ? CallType.Video : CallType.Voice;
this.setState(CallState.Ringing);
@@ -660,9 +622,6 @@ export class MatrixCall extends EventEmitter {
newCall.gotUserMediaForAnswer(this.localAVStream);
delete(this.localAVStream);
}
- newCall.localVideoElement = this.localVideoElement;
- newCall.remoteVideoElement = this.remoteVideoElement;
- newCall.remoteAudioElement = this.remoteAudioElement;
this.successor = newCall;
this.emit(CallEvent.Replaced, newCall);
this.hangup(CallErrorCode.Replaced, true);
@@ -774,10 +733,6 @@ export class MatrixCall extends EventEmitter {
}
this.updateMuteStatus();
- if (!onHold) {
- this.playRemoteAudio();
- }
-
this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold);
}
@@ -831,16 +786,6 @@ export class MatrixCall extends EventEmitter {
const vidShouldBeMuted = this.vidMuted || this.remoteOnHold;
setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted);
-
- if (this.remoteOnHold) {
- if (this.remoteAudioElement && this.remoteAudioElement.srcObject === this.remoteStream) {
- this.remoteAudioElement.muted = true;
- } else if (this.remoteVideoElement && this.remoteVideoElement.srcObject === this.remoteStream) {
- this.remoteVideoElement.muted = true;
- }
- } else {
- this.playRemoteAudio();
- }
}
/**
@@ -863,24 +808,13 @@ export class MatrixCall extends EventEmitter {
logger.debug("gotUserMediaForInvite -> " + this.type);
- const videoEl = this.getLocalVideoElement();
-
- if (videoEl && this.type === CallType.Video) {
- videoEl.autoplay = true;
- if (this.screenSharingStream) {
- logger.debug(
- "Setting screen sharing stream to the local video element",
- );
- videoEl.srcObject = this.screenSharingStream;
- } else {
- videoEl.srcObject = stream;
- }
- videoEl.muted = true;
- try {
- await videoEl.play();
- } catch (e) {
- logger.info("Failed to play local video element", e);
- }
+ if (this.screenSharingStream) {
+ logger.debug(
+ "Setting screen sharing stream to the local video element",
+ );
+ this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), SDPStreamMetadataPurpose.Screenshare);
+ } else {
+ this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia);
}
// why do we enable audio (and only audio) tracks here? -- matthew
@@ -950,19 +884,7 @@ export class MatrixCall extends EventEmitter {
return;
}
- const localVidEl = this.getLocalVideoElement();
-
- if (localVidEl && this.type === CallType.Video) {
- localVidEl.autoplay = true;
- localVidEl.srcObject = stream;
-
- localVidEl.muted = true;
- try {
- await localVidEl.play();
- } catch (e) {
- logger.info("Failed to play local video element", e);
- }
- }
+ this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia);
this.localAVStream = stream;
logger.info("Got local AV stream with id " + this.localAVStream.id);
@@ -1377,32 +1299,31 @@ export class MatrixCall extends EventEmitter {
logger.warn(`Streamless ${ev.track.kind} found: ignoring.`);
return;
}
+
+ const oldRemoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream;
+
// If we already have a stream, check this track is from the same one
- if (this.remoteStream && ev.streams[0].id !== this.remoteStream.id) {
+ // Note that we check by ID and always set the remote stream: Chrome appears
+ // to make new stream objects when tranciever directionality is changed and the 'active'
+ // status of streams change - Dave
+ if (oldRemoteStream && ev.streams[0].id !== oldRemoteStream.id) {
logger.warn(
- `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${this.remoteStream.id}`,
+ `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${oldRemoteStream.id}`,
);
return;
}
- if (!this.remoteStream) {
+ if (!oldRemoteStream) {
logger.info("Got remote stream with id " + ev.streams[0].id);
}
- // Note that we check by ID above and always set the remote stream: Chrome appears
- // to make new stream objects when tranciever directionality is changed and the 'active'
- // status of streams change
- this.remoteStream = ev.streams[0];
+ const newRemoteStream = ev.streams[0];
logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`);
- if (ev.track.kind === 'video') {
- if (this.remoteVideoElement) {
- this.playRemoteVideo();
- }
- } else {
- if (this.remoteAudioElement) this.playRemoteAudio();
- }
+ this.pushNewFeed(newRemoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia)
+
+ logger.info("playing remote. stream active? " + newRemoteStream.active);
};
onNegotiationNeeded = async () => {
@@ -1425,52 +1346,6 @@ export class MatrixCall extends EventEmitter {
}
};
- async playRemoteAudio() {
- if (this.remoteVideoElement) this.remoteVideoElement.muted = true;
- this.remoteAudioElement.muted = false;
-
- this.remoteAudioElement.srcObject = this.remoteStream;
-
- // if audioOutput is non-default:
- try {
- if (audioOutput) {
- // This seems quite unreliable in Chrome, although I haven't yet managed to make a jsfiddle where
- // it fails.
- // It seems reliable if you set the sink ID after setting the srcObject and then set the sink ID
- // back to the default after the call is over
- logger.info("Setting audio sink to " + audioOutput + ", was " + this.remoteAudioElement.sinkId);
- await this.remoteAudioElement.setSinkId(audioOutput);
- }
- } catch (e) {
- logger.warn("Couldn't set requested audio output device: using default", e);
- }
-
- try {
- await this.remoteAudioElement.play();
- } catch (e) {
- logger.error("Failed to play remote audio element", e);
- }
- }
-
- private async playRemoteVideo() {
- // A note on calling methods on media elements:
- // We used to have queues per media element to serialise all calls on those elements.
- // The reason given for this was that load() and play() were racing. However, we now
- // never call load() explicitly so this seems unnecessary. However, serialising every
- // operation was causing bugs where video would not resume because some play command
- // had got stuck and all media operations were queued up behind it. If necessary, we
- // should serialise the ones that need to be serialised but then be able to interrupt
- // them with another load() which will cancel the pending one, but since we don't call
- // load() explicitly, it shouldn't be a problem.
- this.remoteVideoElement.srcObject = this.remoteStream;
- logger.info("playing remote video. stream active? " + this.remoteStream.active);
- try {
- await this.remoteVideoElement.play();
- } catch (e) {
- logger.info("Failed to play remote video element", e);
- }
- }
-
onHangupReceived = (msg) => {
logger.debug("Hangup received for call ID " + this.callId);
@@ -1630,33 +1505,13 @@ export class MatrixCall extends EventEmitter {
this.inviteTimeout = null;
}
- const remoteVid = this.getRemoteVideoElement();
- const remoteAud = this.getRemoteAudioElement();
- const localVid = this.getLocalVideoElement();
+ // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds()
+ this.stopAllMedia();
+ this.deleteAllFeeds();
- if (remoteVid) {
- remoteVid.pause();
- remoteVid.srcObject = null;
- }
- if (remoteAud) {
- remoteAud.pause();
- remoteAud.srcObject = null;
- try {
- // As per comment in playRemoteAudio, setting the sink ID back to the default
- // once the call is over makes setSinkId work reliably.
- await this.remoteAudioElement.setSinkId('')
- } catch (e) {
- logger.warn("Failed to set sink ID back to default");
- }
- }
- if (localVid) {
- localVid.pause();
- localVid.srcObject = null;
- }
this.hangupParty = hangupParty;
this.hangupReason = hangupReason;
this.setState(CallState.Ended);
- this.stopAllMedia();
if (this.peerConn && this.peerConn.signalingState !== 'closed') {
this.peerConn.close();
}
@@ -1667,19 +1522,9 @@ export class MatrixCall extends EventEmitter {
private stopAllMedia() {
logger.debug(`stopAllMedia (stream=${this.localAVStream})`);
- if (this.localAVStream) {
- for (const track of this.localAVStream.getTracks()) {
- track.stop();
- }
- }
- if (this.screenSharingStream) {
- for (const track of this.screenSharingStream.getTracks()) {
- track.stop();
- }
- }
- if (this.remoteStream) {
- for (const track of this.remoteStream.getTracks()) {
+ for (const feed of this.feeds) {
+ for (const track of feed.stream.getTracks()) {
track.stop();
}
}
@@ -1910,16 +1755,8 @@ async function getScreenshareContraints(selectDesktopCapturerSource?: () => Prom
}
}
-let audioOutput: string;
let audioInput: string;
let videoInput: string;
-/**
- * Set an audio output device to use for MatrixCalls
- * @function
- * @param {string=} deviceId the identifier for the device
- * undefined treated as unset
- */
-export function setAudioOutput(deviceId: string) { audioOutput = deviceId; }
/**
* Set an audio input device to use for MatrixCalls
* @function
diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts
index c68903bb8..dce146485 100644
--- a/src/webrtc/callEventTypes.ts
+++ b/src/webrtc/callEventTypes.ts
@@ -1,6 +1,11 @@
// allow camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */
+export enum SDPStreamMetadataPurpose {
+ Usermedia = "m.usermedia",
+ Screenshare = "m.screenshare",
+}
+
interface CallOfferAnswer {
type: string;
sdp: string;
diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts
new file mode 100644
index 000000000..b0004639f
--- /dev/null
+++ b/src/webrtc/callFeed.ts
@@ -0,0 +1,84 @@
+/*
+Copyright 2021 Šimon Brandner