diff --git a/lib/webrtc/call.js b/lib/webrtc/call.js new file mode 100644 index 000000000..c673eaff9 --- /dev/null +++ b/lib/webrtc/call.js @@ -0,0 +1,827 @@ +"use strict"; +var utils = require("../utils"); +var EventEmitter = require("events").EventEmitter; + +// events: onHangup, callPlaced + +/** + * Construct a new Matrix Call. + * @constructor + * @param {Object} opts Config options. + * @param {string} opts.roomId The room ID for this call. + * @param {MatrixClient} opts.client The Matrix Client instance to send events to. + */ +function MatrixCall(opts) { + this.roomId = opts.roomId; + this.client = opts.client; + this.webRtc = opts.webRtc; + // Array of Objects with urls, username, credential keys + this.turnServers = opts.turnServers || [{ + urls: [MatrixCall.FALLBACK_STUN_SERVER] + }]; + utils.forEach(this.turnServers, function(server) { + utils.checkObjectHasKeys(server, ["urls"]); + }); + this.URL = opts.URL; + + this.callId = "c" + new Date().getTime(); + this.state = 'fledgling'; + this.didConnect = false; + + // A queue for candidates waiting to go out. + // We try to amalgamate candidates into a single candidate message where + // possible + this.candidateSendQueue = []; + this.candidateSendTries = 0; +} +/** The length of time a call can be ringing for. */ +MatrixCall.CALL_TIMEOUT_MS = 60000; +/** The fallback server to use for STUN. */ +MatrixCall.FALLBACK_STUN_SERVER = 'stun:stun.l.google.com:19302'; + +utils.inherits(MatrixCall, EventEmitter); + +/** + * Place a voice call to this room. + */ +MatrixCall.prototype.placeVoiceCall = function() { + _placeCallWithConstraints(this, _getUserMediaVideoContraints('voice')); + this.type = 'voice'; +}; + +/** + * Place a video call to this room. + */ +MatrixCall.prototype.placeVideoCall = function() { + _placeCallWithConstraints(this, _getUserMediaVideoContraints('video')); + this.type = 'video'; +}; + +/** + * Retrieve the local video DOM element. + * @return {Element} The dom element + */ +MatrixCall.prototype.getLocalVideoElement = function() { + return this.localVideoSelector; +}; + +/** + * Retrieve the remote video DOM element. + * @return {Element} The dom element + */ +MatrixCall.prototype.getRemoteVideoElement = function() { + return this.remoteVideoSelector; +}; + +/** + * Configure this call from an invite event. + * @param {MatrixEvent} event The m.call.invite event + */ +MatrixCall.prototype.initWithInvite = function(event) { + this.msg = event.getContent(); + this.peerConn = _createPeerConnection(this); + var self = this; + if (this.peerConn) { + this.peerConn.setRemoteDescription( + new this.webRtc.RtcSessionDescription(this.msg.offer), + function(s) { + self.onSetRemoteDescriptionSuccess(s); + }, + function(e) { + self.onSetRemoteDescriptionError(e); + } + ); + } + this.state = 'ringing'; + this.direction = 'inbound'; + + // firefox and Safari's RTCPeerConnection doesn't add streams until it + // starts getting media on them so we need to figure out whether a video + // channel has been offered by ourselves. + if (this.msg.offer.sdp.indexOf('m=video') > -1) { + this.type = 'video'; + } + else { + this.type = 'voice'; + } + + if (event.getAge()) { + setTimeout(function() { + if (self.state == 'ringing') { + self.state = 'ended'; + self.hangupParty = 'remote'; // effectively + stopAllMedia(self); + if (self.peerConn.signalingState != 'closed') { + self.peerConn.close(); + } + self.emit("onHangup", self); + } + }, this.msg.lifetime - event.getAge()); + } +}; + +/** + * Configure this call from a hangup event. + * @param {MatrixEvent} event The m.call.hangup event + */ +MatrixCall.prototype.initWithHangup = function(event) { + // perverse as it may seem, sometimes we want to instantiate a call with a + // hangup message (because when getting the state of the room on load, events + // come in reverse order and we want to remember that a call has been hung up) + this.msg = event.getContent(); + this.state = 'ended'; +}; + +/** + * Answer a call. + */ +MatrixCall.prototype.answer = function() { + console.log("Answering call " + this.callId); + var self = this; + + if (!this.localAVStream && !this.waitForLocalAVStream) { + this.webRtc.getUserMedia( + _getUserMediaVideoContraints(this.type), + function(stream) { + gotUserMediaForAnswer(self, stream); + }, + this.getUserMediaFailed + ); + this.state = 'wait_local_media'; + } else if (this.localAVStream) { + gotUserMediaForAnswer(this, this.localAVStream); + } else if (this.waitForLocalAVStream) { + this.state = 'wait_local_media'; + } +}; + +/** + * Replace this call with a new call, e.g. for glare resolution. + * @param {MatrixCall} newCall The new call. + */ +MatrixCall.prototype.replacedBy = function(newCall) { + console.log(this.callId + " being replaced by " + newCall.callId); + if (this.state == 'wait_local_media') { + console.log("Telling new call to wait for local media"); + newCall.waitForLocalAVStream = true; + } else if (this.state == 'create_offer') { + console.log("Handing local stream to new call"); + gotUserMediaForAnswer(newCall, this.localAVStream); + delete(this.localAVStream); + } else if (this.state == 'invite_sent') { + console.log("Handing local stream to new call"); + gotUserMediaForAnswer(newCall, this.localAVStream); + delete(this.localAVStream); + } + newCall.localVideoSelector = this.localVideoSelector; + newCall.remoteVideoSelector = this.remoteVideoSelector; + this.successor = newCall; + this.hangup(true); +}; + +/** + * Hangup a call. + * @param {string} reason The reason why the call is being hung up. + * @param {boolean} suppressEvent True to suppress emitting an event. + */ +MatrixCall.prototype.hangup = function(reason, suppressEvent) { + console.log("Ending call " + this.callId); + + // pausing now keeps the last frame (ish) of the video call in the video element + // rather than it just turning black straight away + if (this.getRemoteVideoElement() && this.getRemoteVideoElement().pause) { + this.getRemoteVideoElement().pause(); + } + if (this.getLocalVideoElement() && this.getLocalVideoElement().pause) { + this.getLocalVideoElement().pause(); + } + + this.stopAllMedia(); + if (this.peerConn) { + this.peerConn.close(); + } + + this.hangupParty = 'local'; + this.hangupReason = reason; + + var content = { + version: 0, + call_id: this.callId, + reason: reason + }; + this.sendEvent('m.call.hangup', content); + this.state = 'ended'; + if (!suppressEvent) { + this.emit("onHangup", this); + } +}; + +/** + * Internal + * @param {Object} stream + */ +MatrixCall.prototype.gotUserMediaForInvite = function(stream) { + if (this.successor) { + gotUserMediaForAnswer(this.successor, stream); + return; + } + if (this.state == 'ended') { + return; + } + var self = this; + var videoEl = this.getLocalVideoElement(); + + if (videoEl && this.type == 'video') { + videoEl.autoplay = true; + videoEl.src = this.URL.createObjectURL(stream); + videoEl.muted = true; + setTimeout(function() { + var vel = self.getLocalVideoElement(); + if (vel.play) { + vel.play(); + } + }, 0); + } + + this.localAVStream = stream; + var audioTracks = stream.getAudioTracks(); + for (var i = 0; i < audioTracks.length; i++) { + audioTracks[i].enabled = true; + } + this.peerConn = this._createPeerConnection(); + this.peerConn.addStream(stream); + this.peerConn.createOffer(function(d) { + self.gotLocalOffer(d); + }, function(e) { + self.getLocalOfferFailed(e); + }); + self.state = 'create_offer'; +}; + +var gotUserMediaForAnswer = function(self, stream) { + if (self.state == 'ended') { + return; + } + var localVidEl = self.getLocalVideoElement(); + + if (localVidEl && self.type == 'video') { + localVidEl.autoplay = true; + localVidEl.src = self.URL.createObjectURL(stream); + localVidEl.muted = self; + setTimeout(function() { + var vel = self.getLocalVideoElement(); + if (vel.play) { + vel.play(); + } + }, 0); + } + + self.localAVStream = stream; + var audioTracks = stream.getAudioTracks(); + for (var i = 0; i < audioTracks.length; i++) { + audioTracks[i].enabled = true; + } + self.peerConn.addStream(stream); + + var constraints = { + 'mandatory': { + 'OfferToReceiveAudio': true, + 'OfferToReceiveVideo': self.type == 'video' + }, + }; + self.peerConn.createAnswer(constraints, function(description) { + console.log("Created answer: " + description); + self.peerConn.setLocalDescription(description, function() { + var content = { + version: 0, + call_id: self.callId, + answer: { + sdp: self.peerConn.localDescription.sdp, + type: self.peerConn.localDescription.type + } + }; + self.sendEvent('m.call.answer', content); + self.state = 'connecting'; + }, function() { + console.log("Error setting local description!"); + }); + }); + self.state = 'create_answer'; +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.gotLocalIceCandidate = function(event) { + if (event.candidate) { + console.log( + "Got local ICE " + event.candidate.sdpMid + " candidate: " + + event.candidate.candidate + ); + // As with the offer, note we need to make a copy of this object, not + // pass the original: that broke in Chrome ~m43. + var c = { + candidate: event.candidate.candidate, + sdpMid: event.candidate.sdpMid, + sdpMLineIndex: event.candidate.sdpMLineIndex + }; + this.sendCandidate(c); + } +}; + +/** + * Internal + * @param {Object} cand + */ +MatrixCall.prototype.gotRemoteIceCandidate = function(cand) { + if (this.state == 'ended') { + //console.log("Ignoring remote ICE candidate because call has ended"); + return; + } + console.log("Got remote ICE " + cand.sdpMid + " candidate: " + cand.candidate); + this.peerConn.addIceCandidate( + new this.webRtc.RtcIceCandidate(cand), + function() {}, + function(e) {} + ); +}; + +/** + * Internal + * @param {Object} msg + */ +MatrixCall.prototype.receivedAnswer = function(msg) { + if (this.state == 'ended') { + return; + } + + var self = this; + this.peerConn.ngsetRemoteDescription( + new this.webRtc.RtcSessionDescription(msg.answer) + ).then(function(s) { + self.onSetRemoteDescriptionSuccess(s); + }, + function(e) { + self.onSetRemoteDescriptionError(e); + }); + this.state = 'connecting'; +}; + +/** + * Internal + * @param {Object} description + */ +MatrixCall.prototype.gotLocalOffer = function(description) { + var self = this; + console.log("Created offer: " + description); + + if (self.state == 'ended') { + console.log("Ignoring newly created offer on call ID " + self.callId + + " because the call has ended"); + return; + } + + self.peerConn.ngsetLocalDescription(description).then(function() { + var content = { + version: 0, + call_id: self.callId, + // OpenWebRTC appears to add extra stuff (like the DTLS fingerprint) + // to the description when setting it on the peerconnection. + // According to the spec it should only add ICE + // candidates. Any ICE candidates that have already been generated + // at this point will probably be sent both in the offer and separately. + // Also, note that we have to make a new object here, copying the + // type and sdp properties. + // Passing the RTCSessionDescription object as-is doesn't work in + // Chrome (as of about m43). + offer: { + sdp: self.peerConn.localDescription.sdp, + type: self.peerConn.localDescription.type + }, + lifetime: MatrixCall.CALL_TIMEOUT_MS + }; + self.sendEvent('m.call.invite', content); + + setTimeout(function() { + if (self.state == 'invite_sent') { + self.hangup('invite_timeout'); + } + }, MatrixCall.CALL_TIMEOUT_MS); + self.state = 'invite_sent'; + }, function() { + console.log("Error setting local description!"); + } + ); +}; + +/** + * Internal + * @param {Object} error + */ +MatrixCall.prototype.getLocalOfferFailed = function(error) { + this.onError("Failed to start audio for call!"); +}; + +/** + * Internal + */ +MatrixCall.prototype.getUserMediaFailed = function() { + this.onError( + "Couldn't start capturing media! Is your microphone set up and does " + + "this app have permission?" + ); + this.hangup(); +}; + +/** + * Internal + */ +MatrixCall.prototype.onIceConnectionStateChanged = function() { + if (this.state == 'ended') { + return; // because ICE can still complete as we're ending the call + } + console.log( + "Ice connection state changed to: " + this.peerConn.iceConnectionState + ); + // ideally we'd consider the call to be connected when we get media but + // chrome doesn't implement any of the 'onstarted' events yet + if (this.peerConn.iceConnectionState == 'completed' || + this.peerConn.iceConnectionState == 'connected') { + this.state = 'connected'; + this.didConnect = true; + } else if (this.peerConn.iceConnectionState == 'failed') { + this.hangup('ice_failed'); + } +}; + +/** + * Internal + */ +MatrixCall.prototype.onSignallingStateChanged = function() { + console.log( + "call " + this.callId + ": Signalling state changed to: " + + this.peerConn.signalingState + ); +}; + +/** + * Internal + */ +MatrixCall.prototype.onSetRemoteDescriptionSuccess = function() { + console.log("Set remote description"); +}; + +/** + * Internal + * @param {Object} e + */ +MatrixCall.prototype.onSetRemoteDescriptionError = function(e) { + console.log("Failed to set remote description" + e); +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.onAddStream = function(event) { + console.log("Stream added" + event); + + var s = event.stream; + + this.remoteAVStream = s; + + if (this.direction == 'inbound') { + if (s.getVideoTracks().length > 0) { + this.type = 'video'; + } else { + this.type = 'voice'; + } + } + + var self = this; + forAllTracksOnStream(s, function(t) { + // not currently implemented in chrome + t.onstarted = self.onRemoteStreamTrackStarted; + }); + + event.stream.onended = function(e) { self.onRemoteStreamEnded(e); }; + // not currently implemented in chrome + event.stream.onstarted = function(e) { self.onRemoteStreamStarted(e); }; + + this.tryPlayRemoteStream(); +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.tryPlayRemoteStream = function(event) { + if (this.getRemoteVideoElement() && this.remoteAVStream) { + var player = this.getRemoteVideoElement(); + player.autoplay = true; + player.src = this.URL.createObjectURL(this.remoteAVStream); + var self = this; + setTimeout(function() { + var vel = self.getRemoteVideoElement(); + if (vel.play) { + vel.play(); + } + // OpenWebRTC does not support oniceconnectionstatechange yet + if (self.webRtc.isOpenWebRTC()) { + self.state = 'connected'; + } + }, 0); + } +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.onRemoteStreamStarted = function(event) { + this.state = 'connected'; +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.onRemoteStreamEnded = function(event) { + console.log("Remote stream ended"); + this.state = 'ended'; + this.hangupParty = 'remote'; + stopAllMedia(this); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit("onHangup", this); +}; + +/** + * Internal + * @param {Object} event + */ +MatrixCall.prototype.onRemoteStreamTrackStarted = function(event) { + this.state = 'connected'; +}; + +/** + * Internal + * @param {Object} msg + */ +MatrixCall.prototype.onHangupReceived = function(msg) { + console.log("Hangup received"); + if (this.getRemoteVideoElement() && this.getRemoteVideoElement().pause) { + this.getRemoteVideoElement().pause(); + } + if (this.getLocalVideoElement() && this.getLocalVideoElement().pause) { + this.getLocalVideoElement().pause(); + } + this.state = 'ended'; + this.hangupParty = 'remote'; + this.hangupReason = msg.reason; + stopAllMedia(this); + if (this.peerConn && this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit("onHangup", this); +}; + +/** + * Internal + * @param {Object} msg + */ +MatrixCall.prototype.onAnsweredElsewhere = function(msg) { + console.log("Answered elsewhere"); + if (this.getRemoteVideoElement() && this.getRemoteVideoElement().pause) { + this.getRemoteVideoElement().pause(); + } + if (this.getLocalVideoElement() && this.getLocalVideoElement().pause) { + this.getLocalVideoElement().pause(); + } + this.state = 'ended'; + this.hangupParty = 'remote'; + this.hangupReason = "answered_elsewhere"; + stopAllMedia(this); + if (this.peerConn && this.peerConn.signalingState != 'closed') { + this.peerConn.close(); + } + this.emit("onHangup", this); +}; + +/** + * Internal + * @param {string} eventType + * @param {Object} content + * @return {Promise} + */ +MatrixCall.prototype.sendEvent = function(eventType, content) { + return this.client.sendEvent(this.roomId, eventType, content); +}; + +/** + * Internal + * @param {Object} content + */ +MatrixCall.prototype.sendCandidate = function(content) { + // Sends candidates with are sent in a special way because we try to amalgamate + // them into one message + this.candidateSendQueue.push(content); + var self = this; + if (this.candidateSendTries === 0) { + setTimeout(function() { + _sendCandidateQueue(self); + }, 100); + } +}; + +var stopAllMedia = function(self) { + if (self.localAVStream) { + forAllTracksOnStream(self.localAVStream, function(t) { + if (t.stop) { + t.stop(); + } + }); + // also call stop on the main stream so firefox will stop sharing + // the mic + if (self.localAVStream.stop) { + self.localAVStream.stop(); + } + } + if (self.remoteAVStream) { + forAllTracksOnStream(self.remoteAVStream, function(t) { + if (t.stop) { + t.stop(); + } + }); + } +}; + +var _sendCandidateQueue = function(self) { + if (self.candidateSendQueue.length === 0) { + return; + } + + var cands = self.candidateSendQueue; + self.candidateSendQueue = []; + ++self.candidateSendTries; + var content = { + version: 0, + call_id: self.callId, + candidates: cands + }; + console.log("Attempting to send " + cands.length + " candidates"); + self.sendEvent('m.call.candidates', content).then(function() { + self.candidateSendTries = 0; + _sendCandidateQueue(self); + }, function(error) { + for (var i = 0; i < cands.length; i++) { + self.candidateSendQueue.push(cands[i]); + } + + if (self.candidateSendTries > 5) { + console.log( + "Failed to send candidates on attempt %s. Giving up for now.", + self.candidateSendTries + ); + self.candidateSendTries = 0; + return; + } + + var delayMs = 500 * Math.pow(2, self.candidateSendTries); + ++self.candidateSendTries; + console.log("Failed to send candidates. Retrying in " + delayMs + "ms"); + setTimeout(function() { + _sendCandidateQueue(self); + }, delayMs); + }); +}; + +var _placeCallWithConstraints = function(self, constraints) { + self.emit("callPlaced", self); + self.webRtc.getUserMedia( + constraints, self.gotUserMediaForInvite, self.getUserMediaFailed + ); + self.state = 'wait_local_media'; + self.direction = 'outbound'; + self.config = constraints; +}; + +var _createPeerConnection = function(self) { + var servers = self.turnServers; + if (self.webRtc.vendor === "mozilla") { + // modify turnServers struct to match what mozilla expects. + servers = []; + for (var i = 0; i < self.turnServers.length; i++) { + for (var j = 0; j < self.turnServers[i].urls.length; j++) { + servers.push({ + url: self.turnServers[i].urls[j], + username: self.turnServers[i].username, + credential: self.turnServers[i].credential + }); + } + } + } + + var pc = new self.webRtc.RtcPeerConnection({ + iceServers: servers + }); + pc.oniceconnectionstatechange = self.onIceConnectionStateChanged; + pc.onsignalingstatechange = self.onSignallingStateChanged; + pc.onicecandidate = self.gotLocalIceCandidate; + pc.onaddstream = self.onAddStream; + return pc; +}; + +var _getUserMediaVideoContraints = function(callType) { + switch (callType) { + case 'voice': + return ({audio: true, video: false}); + case 'video': + return ({audio: true, video: { + mandatory: { + minWidth: 640, + maxWidth: 640, + minHeight: 360, + maxHeight: 360, + } + }}); + } +}; + +var forAllVideoTracksOnStream = function(s, f) { + var tracks = s.getVideoTracks(); + for (var i = 0; i < tracks.length; i++) { + f(tracks[i]); + } +}; + +var forAllAudioTracksOnStream = function(s, f) { + var tracks = s.getAudioTracks(); + for (var i = 0; i < tracks.length; i++) { + f(tracks[i]); + } +}; + +var forAllTracksOnStream = function(s, f) { + forAllVideoTracksOnStream(s, f); + forAllAudioTracksOnStream(s, f); +}; + +/** The MatrixCall class. */ +module.exports.MatrixCall = MatrixCall; + +/** + * Create a new Matrix call for the browser. + * @param {MatrixClient} client The client instance to use. + * @param {string} roomId The room the call is in. + * @return {MatrixCall} the call or null if the browser doesn't support calling. + */ +module.exports.createNewMatrixCall = function(client, roomId) { + var w = global.window; + var webRtc = {}; + webRtc.isOpenWebRTC = function() { + // TODO + }; + var getUserMedia = ( + w.navigator.getUserMedia || w.navigator.webkitGetUserMedia || + w.navigator.mozGetUserMedia + ); + if (getUserMedia) { + webRtc.getUserMedia = function() { + return getUserMedia.apply(w.navigator, arguments); + }; + } + webRtc.RtcPeerConnection = ( + w.RTCPeerConnection || w.webkitRTCPeerConnection || w.mozRTCPeerConnection + ); + webRtc.RtcSessionDescription = ( + w.RTCSessionDescription || w.webkitRTCSessionDescription || + w.mozRTCSessionDescription + ); + webRtc.RtcIceCandidate = ( + w.RTCIceCandidate || w.webkitRTCIceCandidate || w.mozRTCIceCandidate + ); + webRtc.vendor = null; + if (w.mozRTCPeerConnection) { + webRtc.vendor = "mozilla"; + } + else if (w.webkitRTCPeerConnection) { + webRtc.vendor = "webkit"; + } + else if (w.RTCPeerConnection) { + webRtc.vendor = "generic"; + } + if (!webRtc.RtcIceCandidate || !webRtc.RtcSessionDescription || + !webRtc.RtcPeerConnection || !webRtc.getUserMedia) { + return null; // Web RTC is not supported. + } + var opts = { + webRtc: webRtc, + client: client, + URL: w.URL, + roomId: roomId + }; + return new MatrixCall(opts); +};