matrix/matrix-js-sdk
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-js-sdk.git synced 2025-07-30 04:23:07 +03:00
Code Activity

Refactor GroupCall participant management

Browse Source 
This refactoring brings a number of improvements to GroupCall, which I've unfortunately had to combine into a single commit due to coupling:

- Moves the expiration timestamp field on call membership state to be per-device
- Makes the participants of a group call visible without having to enter the call yourself
- Enables users to join group calls from multiple devices
- Identifies active speakers by their call feed, rather than just their user ID
- Plays nicely with clients that can be in multiple calls in a room at once
- Fixes a memory leak caused by the call retry loop never stopping
- Changes GroupCall to update its state synchronously, and write back to room state asynchronously
  - This was already sort of halfway being done, but now we'd be committing to it
  - Generally improves the robustness of the state machine
  - It means that group call joins will appear instant, in a sense

For many reasons, this is a breaking change.
This commit is contained in:
Robin Townsend
2022-11-21 11:58:27 -05:00
parent dd98d7eb2c
commit f46ecf970c
10 changed files with 735 additions and 668 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
spec/test-utils/webrtc.ts
View File

@ -431,6 +431,7 @@ export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, Emitt
export class MockCallFeed { export class MockCallFeed {
constructor( constructor(
public userId: string, public userId: string,
public deviceId: string | undefined,
public stream: MockMediaStream, public stream: MockMediaStream,
) {} ) {}

33
spec/unit/webrtc/call.spec.ts
View File

@ -81,10 +81,13 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s
call.getFeeds().push(new CallFeed({ call.getFeeds().push(new CallFeed({
client: client.client, client: client.client,
userId: "remote_user_id", userId: "remote_user_id",
// @ts-ignore Mock deviceId: undefined,
stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), stream: new MockMediaStream(
id: "remote_feed_id", "remote_stream_id", [new MockMediaStreamTrack("remote_tack_id", "audio")],
) as unknown as MediaStream,
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false,
videoMuted: false,
})); }));
await callPromise; await callPromise;
}; };
@ -447,7 +450,7 @@ describe('Call', function() {
client.client.getRoom = () => { client.client.getRoom = () => {
return { return {
getMember: (userId) => { getMember: (userId: string) => {
if (userId === opponentMember.userId) { if (userId === opponentMember.userId) {
return opponentMember; return opponentMember;
} }
@ -521,10 +524,12 @@ describe('Call', function() {
it("should correctly generate local SDPStreamMetadata", async () => { it("should correctly generate local SDPStreamMetadata", async () => {
const callPromise = call.placeCallWithCallFeeds([new CallFeed({ const callPromise = call.placeCallWithCallFeeds([new CallFeed({
client: client.client, client: client.client,
// @ts-ignore Mock stream: new MockMediaStream(
stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]), "local_stream1", [new MockMediaStreamTrack("track_id", "audio")],
) as unknown as MediaStream,
roomId: call.roomId, roomId: call.roomId,
userId: client.getUserId(), userId: client.getUserId(),
deviceId: undefined,
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false, audioMuted: false,
videoMuted: false, videoMuted: false,
@ -534,8 +539,10 @@ describe('Call', function() {
call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" });
(call as any).pushNewLocalFeed( (call as any).pushNewLocalFeed(
new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]), new MockMediaStream(
SDPStreamMetadataPurpose.Screenshare, "feed_id2", "local_stream2", [new MockMediaStreamTrack("track_id", "video")],
) as unknown as MediaStream,
SDPStreamMetadataPurpose.Screenshare,
); );
await call.setMicrophoneMuted(true); await call.setMicrophoneMuted(true);
@ -563,20 +570,18 @@ describe('Call', function() {
new CallFeed({ new CallFeed({
client: client.client, client: client.client,
userId: client.getUserId(), userId: client.getUserId(),
// @ts-ignore Mock deviceId: undefined,
stream: localUsermediaStream, stream: localUsermediaStream as unknown as MediaStream,
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
id: "local_usermedia_feed_id",
audioMuted: false, audioMuted: false,
videoMuted: false, videoMuted: false,
}), }),
new CallFeed({ new CallFeed({
client: client.client, client: client.client,
userId: client.getUserId(), userId: client.getUserId(),
// @ts-ignore Mock deviceId: undefined,
stream: localScreensharingStream, stream: localScreensharingStream as unknown as MediaStream,
purpose: SDPStreamMetadataPurpose.Screenshare, purpose: SDPStreamMetadataPurpose.Screenshare,
id: "local_screensharing_feed_id",
audioMuted: false, audioMuted: false,
videoMuted: false, videoMuted: false,
}), }),

207
spec/unit/webrtc/groupCall.spec.ts
View File

@ -23,6 +23,7 @@ import {
Room, Room,
RoomMember, RoomMember,
} from '../../../src'; } from '../../../src';
import { RoomStateEvent } from "../../../src/models/room-state";
import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall"; import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall";
import { MatrixClient } from "../../../src/client"; import { MatrixClient } from "../../../src/client";
import { import {
@ -53,18 +54,19 @@ const FAKE_USER_ID_3 = "@charlie:test.dummy";
const FAKE_STATE_EVENTS = [ const FAKE_STATE_EVENTS = [
{ {
getContent: () => ({ getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR, "m.calls": [],
}), }),
getStateKey: () => FAKE_USER_ID_1, getStateKey: () => FAKE_USER_ID_1,
getRoomId: () => FAKE_ROOM_ID, getRoomId: () => FAKE_ROOM_ID,
}, },
{ {
getContent: () => ({ getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR, "m.calls": [{
["m.calls"]: [{ "m.call_id": FAKE_CONF_ID,
["m.call_id"]: FAKE_CONF_ID, "m.devices": [{
["m.devices"]: [{
device_id: FAKE_DEVICE_ID_2, device_id: FAKE_DEVICE_ID_2,
session_id: FAKE_SESSION_ID_2,
expires_ts: Date.now() + ONE_HOUR,
feeds: [], feeds: [],
}], }],
}], }],
@ -73,11 +75,13 @@ const FAKE_STATE_EVENTS = [
getRoomId: () => FAKE_ROOM_ID, getRoomId: () => FAKE_ROOM_ID,
}, { }, {
getContent: () => ({ getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR, "m.expires_ts": Date.now() + ONE_HOUR,
["m.calls"]: [{ "m.calls": [{
["m.call_id"]: FAKE_CONF_ID, "m.call_id": FAKE_CONF_ID,
["m.devices"]: [{ "m.devices": [{
device_id: "user3_device", device_id: "user3_device",
session_id: "user3_session",
expires_ts: Date.now() + ONE_HOUR,
feeds: [], feeds: [],
}], }],
}], }],
@ -111,6 +115,8 @@ class MockCall {
public state = CallState.Ringing; public state = CallState.Ringing;
public opponentUserId = FAKE_USER_ID_1; public opponentUserId = FAKE_USER_ID_1;
public opponentDeviceId = FAKE_DEVICE_ID_1;
public opponentMember = { userId: this.opponentUserId };
public callId = "1"; public callId = "1";
public localUsermediaFeed = { public localUsermediaFeed = {
setAudioVideoMuted: jest.fn<void, [boolean, boolean]>(), setAudioVideoMuted: jest.fn<void, [boolean, boolean]>(),
@ -129,9 +135,11 @@ class MockCall {
public removeListener = jest.fn(); public removeListener = jest.fn();
public getOpponentMember(): Partial<RoomMember> { public getOpponentMember(): Partial<RoomMember> {
return { return this.opponentMember;
userId: this.opponentUserId, }
};
public getOpponentDeviceId(): string {
return this.opponentDeviceId;
} }
public typed(): MatrixCall { return this as unknown as MatrixCall; } public typed(): MatrixCall { return this as unknown as MatrixCall; }
@ -326,8 +334,8 @@ describe('Group Call', function() {
describe("call feeds changing", () => { describe("call feeds changing", () => {
let call: MockCall; let call: MockCall;
const currentFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("current")); const currentFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("current"));
const newFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("new")); const newFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("new"));
beforeEach(async () => { beforeEach(async () => {
jest.spyOn(currentFeed, "dispose"); jest.spyOn(currentFeed, "dispose");
@ -358,7 +366,7 @@ describe('Group Call', function() {
}); });
it("replaces usermedia feed", async () => { it("replaces usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()]; groupCall.userMediaFeeds.push(currentFeed.typed());
call.remoteUsermediaFeed = newFeed.typed(); call.remoteUsermediaFeed = newFeed.typed();
// @ts-ignore Mock // @ts-ignore Mock
@ -368,7 +376,7 @@ describe('Group Call', function() {
}); });
it("removes usermedia feed", async () => { it("removes usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()]; groupCall.userMediaFeeds.push(currentFeed.typed());
// @ts-ignore Mock // @ts-ignore Mock
groupCall.onCallFeedsChanged(call); groupCall.onCallFeedsChanged(call);
@ -387,7 +395,7 @@ describe('Group Call', function() {
}); });
it("replaces screenshare feed", async () => { it("replaces screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()]; groupCall.screenshareFeeds.push(currentFeed.typed());
call.remoteScreensharingFeed = newFeed.typed(); call.remoteScreensharingFeed = newFeed.typed();
// @ts-ignore Mock // @ts-ignore Mock
@ -397,7 +405,7 @@ describe('Group Call', function() {
}); });
it("removes screenshare feed", async () => { it("removes screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()]; groupCall.screenshareFeeds.push(currentFeed.typed());
// @ts-ignore Mock // @ts-ignore Mock
groupCall.onCallFeedsChanged(call); groupCall.onCallFeedsChanged(call);
@ -408,7 +416,7 @@ describe('Group Call', function() {
describe("feed replacing", () => { describe("feed replacing", () => {
it("replaces usermedia feed", async () => { it("replaces usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()]; groupCall.userMediaFeeds.push(currentFeed.typed());
// @ts-ignore Mock // @ts-ignore Mock
groupCall.replaceUserMediaFeed(currentFeed, newFeed); groupCall.replaceUserMediaFeed(currentFeed, newFeed);
@ -422,7 +430,7 @@ describe('Group Call', function() {
}); });
it("replaces screenshare feed", async () => { it("replaces screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()]; groupCall.screenshareFeeds.push(currentFeed.typed());
// @ts-ignore Mock // @ts-ignore Mock
groupCall.replaceScreenshareFeed(currentFeed, newFeed); groupCall.replaceScreenshareFeed(currentFeed, newFeed);
@ -489,7 +497,10 @@ describe('Group Call', function() {
it("sends metadata updates before unmuting in PTT mode", async () => { it("sends metadata updates before unmuting in PTT mode", async () => {
const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId);
groupCall.calls.push(mockCall as unknown as MatrixCall); groupCall.calls.set(
mockCall.getOpponentMember() as RoomMember,
new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]),
);
let metadataUpdateResolve: () => void; let metadataUpdateResolve: () => void;
const metadataUpdatePromise = new Promise<void>(resolve => { const metadataUpdatePromise = new Promise<void>(resolve => {
@ -511,7 +522,10 @@ describe('Group Call', function() {
it("sends metadata updates after muting in PTT mode", async () => { it("sends metadata updates after muting in PTT mode", async () => {
const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId);
groupCall.calls.push(mockCall as unknown as MatrixCall); groupCall.calls.set(
mockCall.getOpponentMember() as RoomMember,
new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]),
);
// the call starts muted, so unmute to get in the right state to test // the call starts muted, so unmute to get in the right state to test
await groupCall.setMicrophoneMuted(false); await groupCall.setMicrophoneMuted(false);
@ -560,7 +574,7 @@ describe('Group Call', function() {
if (eventType === EventType.GroupCallMemberPrefix) { if (eventType === EventType.GroupCallMemberPrefix) {
const fakeEvent = { const fakeEvent = {
getContent: () => content, getContent: () => content,
getRoomId: () => FAKE_ROOM_ID, getRoomId: () => roomId,
getStateKey: () => statekey, getStateKey: () => statekey,
} as unknown as MatrixEvent; } as unknown as MatrixEvent;
@ -574,8 +588,8 @@ describe('Group Call', function() {
// just add it once. // just add it once.
subMap.set(statekey, fakeEvent); subMap.set(statekey, fakeEvent);
groupCall1.onMemberStateChanged(fakeEvent); client1Room.currentState.emit(RoomStateEvent.Update, client1Room.currentState);
groupCall2.onMemberStateChanged(fakeEvent); client2Room.currentState.emit(RoomStateEvent.Update, client2Room.currentState);
} }
return Promise.resolve({ "event_id": "foo" }); return Promise.resolve({ "event_id": "foo" });
}; };
@ -584,9 +598,17 @@ describe('Group Call', function() {
client2.sendStateEvent.mockImplementation(fakeSendStateEvents); client2.sendStateEvent.mockImplementation(fakeSendStateEvents);
const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1); const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1);
const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2); const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2);
client1Room.currentState.members[FAKE_USER_ID_1] = client2Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
client1Room.currentState.members[FAKE_USER_ID_2] = client2Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
groupCall1 = new GroupCall( groupCall1 = new GroupCall(
client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID,
); );
@ -594,20 +616,6 @@ describe('Group Call', function() {
groupCall2 = new GroupCall( groupCall2 = new GroupCall(
client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID,
); );
client1Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
client1Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
} as unknown as RoomMember;
client2Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
client2Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
} as unknown as RoomMember;
}); });
afterEach(function() { afterEach(function() {
@ -672,8 +680,10 @@ describe('Group Call', function() {
expect(client1.sendToDevice).toHaveBeenCalled(); expect(client1.sendToDevice).toHaveBeenCalled();
const oldCall = groupCall1.getCallByUserId(client2.userId); const oldCall = groupCall1.calls.get(
oldCall!.emit(CallEvent.Hangup, oldCall!); groupCall1.room.getMember(client2.userId)!,
)!.get(client2.deviceId)!;
oldCall.emit(CallEvent.Hangup, oldCall!);
client1.sendToDevice.mockClear(); client1.sendToDevice.mockClear();
@ -691,9 +701,11 @@ describe('Group Call', function() {
// to even be created... // to even be created...
let newCall: MatrixCall | undefined; let newCall: MatrixCall | undefined;
while ( while (
(newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || (newCall = groupCall1.calls.get(
newCall.peerConn === undefined || groupCall1.room.getMember(client2.userId)!,
newCall.callId == oldCall!.callId )?.get(client2.deviceId)) === undefined
|| newCall.peerConn === undefined
|| newCall.callId == oldCall.callId
) { ) {
await flushPromises(); await flushPromises();
} }
@ -733,7 +745,9 @@ describe('Group Call', function() {
groupCall1.setMicrophoneMuted(false); groupCall1.setMicrophoneMuted(false);
groupCall1.setLocalVideoMuted(false); groupCall1.setLocalVideoMuted(false);
const call = groupCall1.getCallByUserId(client2.userId)!; const call = groupCall1.calls.get(
groupCall1.room.getMember(client2.userId)!,
)!.get(client2.deviceId)!;
call.isMicrophoneMuted = jest.fn().mockReturnValue(true); call.isMicrophoneMuted = jest.fn().mockReturnValue(true);
call.setMicrophoneMuted = jest.fn(); call.setMicrophoneMuted = jest.fn();
call.isLocalVideoMuted = jest.fn().mockReturnValue(true); call.isLocalVideoMuted = jest.fn().mockReturnValue(true);
@ -765,7 +779,14 @@ describe('Group Call', function() {
? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS
: { getContent: () => ([]) }; : { getContent: () => ([]) };
}); });
room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
}); });
describe("local muting", () => { describe("local muting", () => {
@ -773,17 +794,13 @@ describe('Group Call', function() {
const groupCall = await createAndEnterGroupCall(mockClient, room); const groupCall = await createAndEnterGroupCall(mockClient, room);
groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); groupCall.localCallFeed!.setAudioVideoMuted = jest.fn();
const setAVMutedArray = groupCall.calls.map(call => { const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = [];
call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); const tracksArray: MediaStreamTrack[] = [];
return call.localUsermediaFeed!.setAudioVideoMuted; const sendMetadataUpdateArray: (() => Promise<void>)[] = [];
}); groupCall.forEachCall(call => {
const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn());
acc.push(...call.localUsermediaStream!.getAudioTracks()); tracksArray.push(...call.localUsermediaStream!.getAudioTracks());
return acc; sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn());
}, []);
const sendMetadataUpdateArray = groupCall.calls.map(call => {
call.sendMetadataUpdate = jest.fn();
return call.sendMetadataUpdate;
}); });
await groupCall.setMicrophoneMuted(true); await groupCall.setMicrophoneMuted(true);
@ -801,18 +818,14 @@ describe('Group Call', function() {
const groupCall = await createAndEnterGroupCall(mockClient, room); const groupCall = await createAndEnterGroupCall(mockClient, room);
groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); groupCall.localCallFeed!.setAudioVideoMuted = jest.fn();
const setAVMutedArray = groupCall.calls.map(call => { const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = [];
call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); const tracksArray: MediaStreamTrack[] = [];
const sendMetadataUpdateArray: (() => Promise<void>)[] = [];
groupCall.forEachCall(call => {
call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true); call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true);
return call.localUsermediaFeed!.setAudioVideoMuted; setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn());
}); tracksArray.push(...call.localUsermediaStream!.getVideoTracks());
const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn());
acc.push(...call.localUsermediaStream!.getVideoTracks());
return acc;
}, []);
const sendMetadataUpdateArray = groupCall.calls.map(call => {
call.sendMetadataUpdate = jest.fn();
return call.sendMetadataUpdate;
}); });
await groupCall.setLocalVideoMuted(true); await groupCall.setLocalVideoMuted(true);
@ -847,7 +860,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created // It takes a bit of time for the calls to get created
await sleep(10); await sleep(10);
const call = groupCall.calls[0]; const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
// @ts-ignore Mock // @ts-ignore Mock
call.pushRemoteFeed(new MockMediaStream("stream", [ call.pushRemoteFeed(new MockMediaStream("stream", [
@ -856,7 +869,7 @@ describe('Group Call', function() {
])); ]));
call.onSDPStreamMetadataChangedReceived(metadataEvent); call.onSDPStreamMetadataChangedReceived(metadataEvent);
const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!);
expect(feed!.isAudioMuted()).toBe(true); expect(feed!.isAudioMuted()).toBe(true);
expect(feed!.isVideoMuted()).toBe(false); expect(feed!.isVideoMuted()).toBe(false);
@ -870,7 +883,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created // It takes a bit of time for the calls to get created
await sleep(10); await sleep(10);
const call = groupCall.calls[0]; const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
// @ts-ignore Mock // @ts-ignore Mock
call.pushRemoteFeed(new MockMediaStream("stream", [ call.pushRemoteFeed(new MockMediaStream("stream", [
@ -879,7 +892,7 @@ describe('Group Call', function() {
])); ]));
call.onSDPStreamMetadataChangedReceived(metadataEvent); call.onSDPStreamMetadataChangedReceived(metadataEvent);
const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!);
expect(feed!.isAudioMuted()).toBe(false); expect(feed!.isAudioMuted()).toBe(false);
expect(feed!.isVideoMuted()).toBe(true); expect(feed!.isVideoMuted()).toBe(true);
@ -945,12 +958,16 @@ describe('Group Call', function() {
expect(mockCall.reject).not.toHaveBeenCalled(); expect(mockCall.reject).not.toHaveBeenCalled();
expect(mockCall.answerWithCallFeeds).toHaveBeenCalled(); expect(mockCall.answerWithCallFeeds).toHaveBeenCalled();
expect(groupCall.calls).toEqual([mockCall]); expect(groupCall.calls).toEqual(new Map([[
groupCall.room.getMember(FAKE_USER_ID_1)!,
new Map([[FAKE_DEVICE_ID_1, mockCall]]),
]]));
}); });
it("replaces calls if it already has one with the same user", async () => { it("replaces calls if it already has one with the same user", async () => {
const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId); const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId);
const newMockCall = new MockCall(room.roomId, groupCall.groupCallId); const newMockCall = new MockCall(room.roomId, groupCall.groupCallId);
newMockCall.opponentMember = oldMockCall.opponentMember; // Ensure referential equality
newMockCall.callId = "not " + oldMockCall.callId; newMockCall.callId = "not " + oldMockCall.callId;
mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall); mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall);
@ -958,7 +975,10 @@ describe('Group Call', function() {
expect(oldMockCall.hangup).toHaveBeenCalled(); expect(oldMockCall.hangup).toHaveBeenCalled();
expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled(); expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled();
expect(groupCall.calls).toEqual([newMockCall]); expect(groupCall.calls).toEqual(new Map([[
groupCall.room.getMember(FAKE_USER_ID_1)!,
new Map([[FAKE_DEVICE_ID_1, newMockCall]]),
]]));
}); });
it("starts to process incoming calls when we've entered", async () => { it("starts to process incoming calls when we've entered", async () => {
@ -988,7 +1008,14 @@ describe('Group Call', function() {
mockClient = typedMockClient.typed(); mockClient = typedMockClient.typed();
room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1);
room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => {
return type === EventType.GroupCallMemberPrefix return type === EventType.GroupCallMemberPrefix
? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS
@ -999,21 +1026,20 @@ describe('Group Call', function() {
}); });
it("sending screensharing stream", async () => { it("sending screensharing stream", async () => {
const onNegotiationNeededArray = groupCall.calls.map(call => { const onNegotiationNeededArray: (() => Promise<void>)[] = [];
groupCall.forEachCall(call => {
// @ts-ignore Mock // @ts-ignore Mock
call.gotLocalOffer = jest.fn(); onNegotiationNeededArray.push(call.gotLocalOffer = jest.fn());
// @ts-ignore Mock
return call.gotLocalOffer;
}); });
let enabledResult; let enabledResult: boolean;
enabledResult = await groupCall.setScreensharingEnabled(true); enabledResult = await groupCall.setScreensharingEnabled(true);
expect(enabledResult).toEqual(true); expect(enabledResult).toEqual(true);
expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled(); expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled();
MockRTCPeerConnection.triggerAllNegotiations(); MockRTCPeerConnection.triggerAllNegotiations();
expect(groupCall.screenshareFeeds).toHaveLength(1); expect(groupCall.screenshareFeeds).toHaveLength(1);
groupCall.calls.forEach(c => { groupCall.forEachCall(c => {
expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined(); expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined();
}); });
onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled()); onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled());
@ -1036,7 +1062,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created // It takes a bit of time for the calls to get created
await sleep(10); await sleep(10);
const call = groupCall.calls[0]; const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
call.onNegotiateReceived({ call.onNegotiateReceived({
getContent: () => ({ getContent: () => ({
@ -1057,7 +1083,7 @@ describe('Group Call', function() {
])); ]));
expect(groupCall.screenshareFeeds).toHaveLength(1); expect(groupCall.screenshareFeeds).toHaveLength(1);
expect(groupCall.getScreenshareFeedByUserId(call.invitee!)).toBeDefined(); expect(groupCall.getScreenshareFeed(call.invitee!, call.getOpponentDeviceId()!)).toBeDefined();
groupCall.terminate(); groupCall.terminate();
}); });
@ -1097,12 +1123,16 @@ describe('Group Call', function() {
); );
room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1);
room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
groupCall = await createAndEnterGroupCall(mockClient.typed(), room); groupCall = await createAndEnterGroupCall(mockClient.typed(), room);
mediaFeed1 = new CallFeed({ mediaFeed1 = new CallFeed({
client: mockClient.typed(), client: mockClient.typed(),
roomId: FAKE_ROOM_ID, roomId: FAKE_ROOM_ID,
userId: FAKE_USER_ID_2, userId: FAKE_USER_ID_2,
deviceId: FAKE_DEVICE_ID_1,
stream: (new MockMediaStream("foo", [])).typed(), stream: (new MockMediaStream("foo", [])).typed(),
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false, audioMuted: false,
@ -1114,6 +1144,7 @@ describe('Group Call', function() {
client: mockClient.typed(), client: mockClient.typed(),
roomId: FAKE_ROOM_ID, roomId: FAKE_ROOM_ID,
userId: FAKE_USER_ID_3, userId: FAKE_USER_ID_3,
deviceId: FAKE_DEVICE_ID_1,
stream: (new MockMediaStream("foo", [])).typed(), stream: (new MockMediaStream("foo", [])).typed(),
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false, audioMuted: false,
@ -1136,15 +1167,15 @@ describe('Group Call', function() {
mediaFeed2.speakingVolumeSamples = [0, 0]; mediaFeed2.speakingVolumeSamples = [0, 0];
jest.runOnlyPendingTimers(); jest.runOnlyPendingTimers();
expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_2); expect(groupCall.activeSpeaker).toEqual(mediaFeed1);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_2); expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed1);
mediaFeed1.speakingVolumeSamples = [0, 0]; mediaFeed1.speakingVolumeSamples = [0, 0];
mediaFeed2.speakingVolumeSamples = [100, 100]; mediaFeed2.speakingVolumeSamples = [100, 100];
jest.runOnlyPendingTimers(); jest.runOnlyPendingTimers();
expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_3); expect(groupCall.activeSpeaker).toEqual(mediaFeed2);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3); expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed2);
}); });
}); });

53
spec/unit/webrtc/groupCallEventHandler.spec.ts
View File

@ -16,26 +16,21 @@ limitations under the License.
import { mocked } from "jest-mock"; import { mocked } from "jest-mock";
import { ClientEvent } from "../../../src/client";
import { RoomMember } from "../../../src/models/room-member";
import { SyncState } from "../../../src/sync";
import { import {
ClientEvent,
GroupCall,
GroupCallIntent, GroupCallIntent,
GroupCallState, GroupCallState,
GroupCallType, GroupCallType,
IContent, GroupCallTerminationReason,
MatrixEvent, } from "../../../src/webrtc/groupCall";
Room, import { IContent, MatrixEvent } from "../../../src/models/event";
RoomState, import { Room } from "../../../src/models/room";
} from "../../../src"; import { RoomState } from "../../../src/models/room-state";
import { SyncState } from "../../../src/sync";
import { GroupCallTerminationReason } from "../../../src/webrtc/groupCall";
import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler"; import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler";
import { flushPromises } from "../../test-utils/flushPromises"; import { flushPromises } from "../../test-utils/flushPromises";
import { import { makeMockGroupCallStateEvent, MockCallMatrixClient } from "../../test-utils/webrtc";
makeMockGroupCallMemberStateEvent,
makeMockGroupCallStateEvent,
MockCallMatrixClient,
} from "../../test-utils/webrtc";
const FAKE_USER_ID = "@alice:test.dummy"; const FAKE_USER_ID = "@alice:test.dummy";
const FAKE_DEVICE_ID = "AAAAAAA"; const FAKE_DEVICE_ID = "AAAAAAA";
@ -47,6 +42,7 @@ describe('Group Call Event Handler', function() {
let groupCallEventHandler: GroupCallEventHandler; let groupCallEventHandler: GroupCallEventHandler;
let mockClient: MockCallMatrixClient; let mockClient: MockCallMatrixClient;
let mockRoom: Room; let mockRoom: Room;
let mockMember: RoomMember;
beforeEach(() => { beforeEach(() => {
mockClient = new MockCallMatrixClient( mockClient = new MockCallMatrixClient(
@ -54,13 +50,21 @@ describe('Group Call Event Handler', function() {
); );
groupCallEventHandler = new GroupCallEventHandler(mockClient.typed()); groupCallEventHandler = new GroupCallEventHandler(mockClient.typed());
mockMember = {
userId: FAKE_USER_ID,
membership: "join",
} as unknown as RoomMember;
mockRoom = { mockRoom = {
on: () => {},
off: () => {},
roomId: FAKE_ROOM_ID, roomId: FAKE_ROOM_ID,
currentState: { currentState: {
getStateEvents: jest.fn().mockReturnValue([makeMockGroupCallStateEvent( getStateEvents: jest.fn().mockReturnValue([makeMockGroupCallStateEvent(
FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, FAKE_ROOM_ID, FAKE_GROUP_CALL_ID,
)]), )]),
}, },
getMember: (userId: string) => userId === FAKE_USER_ID ? mockMember : null,
} as unknown as Room; } as unknown as Room;
(mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom);
@ -211,27 +215,6 @@ describe('Group Call Event Handler', function() {
); );
}); });
it("sends member events to group calls", async () => {
await groupCallEventHandler.start();
const mockGroupCall = {
onMemberStateChanged: jest.fn(),
};
groupCallEventHandler.groupCalls.set(FAKE_ROOM_ID, mockGroupCall as unknown as GroupCall);
const mockStateEvent = makeMockGroupCallMemberStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID);
mockClient.emitRoomState(
mockStateEvent,
{
roomId: FAKE_ROOM_ID,
} as unknown as RoomState,
);
expect(mockGroupCall.onMemberStateChanged).toHaveBeenCalledWith(mockStateEvent);
});
describe("ignoring invalid group call state events", () => { describe("ignoring invalid group call state events", () => {
let mockClientEmit: jest.Func; let mockClientEmit: jest.Func;

13
src/utils.ts
View File

@ -694,3 +694,16 @@ export function sortEventsByLatestContentTimestamp(left: MatrixEvent, right: Mat
export function isSupportedReceiptType(receiptType: string): boolean { export function isSupportedReceiptType(receiptType: string): boolean {
return [ReceiptType.Read, ReceiptType.ReadPrivate].includes(receiptType as ReceiptType); return [ReceiptType.Read, ReceiptType.ReadPrivate].includes(receiptType as ReceiptType);
} }
/**
* Determines whether two maps are equal.
* @param eq The equivalence relation to compare values by. Defaults to strict equality.
*/
export function mapsEqual<K, V>(x: Map<K, V>, y: Map<K, V>, eq = (v1: V, v2: V): boolean => v1 === v2): boolean {
if (x.size !== y.size) return false;
for (const [k, v1] of x) {
const v2 = y.get(k);
if (v2 === undefined || !eq(v1, v2)) return false;
}
return true;
}

9
src/webrtc/call.ts
View File

@ -462,6 +462,10 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
return this.opponentMember; return this.opponentMember;
} }
public getOpponentDeviceId(): string | undefined {
return this.opponentDeviceId;
}
public getOpponentSessionId(): string | undefined { public getOpponentSessionId(): string | undefined {
return this.opponentSessionId; return this.opponentSessionId;
} }
@ -644,6 +648,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
client: this.client, client: this.client,
roomId: this.roomId, roomId: this.roomId,
userId, userId,
deviceId: this.getOpponentDeviceId(),
stream, stream,
purpose, purpose,
audioMuted, audioMuted,
@ -688,6 +693,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
audioMuted: false, audioMuted: false,
videoMuted: false, videoMuted: false,
userId, userId,
deviceId: this.getOpponentDeviceId(),
stream, stream,
purpose, purpose,
})); }));
@ -718,6 +724,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
audioMuted: false, audioMuted: false,
videoMuted: false, videoMuted: false,
userId, userId,
deviceId: this.getOpponentDeviceId(),
stream, stream,
purpose, purpose,
}), }),
@ -1009,6 +1016,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
client: this.client, client: this.client,
roomId: this.roomId, roomId: this.roomId,
userId: this.client.getUserId()!, userId: this.client.getUserId()!,
deviceId: this.client.getDeviceId() ?? undefined,
stream, stream,
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false, audioMuted: false,
@ -2584,6 +2592,7 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
client: this.client, client: this.client,
roomId: this.roomId, roomId: this.roomId,
userId: this.client.getUserId()!, userId: this.client.getUserId()!,
deviceId: this.client.getDeviceId() ?? undefined,
stream, stream,
purpose: SDPStreamMetadataPurpose.Usermedia, purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false, audioMuted: false,

4
src/webrtc/callEventHandler.ts
View File

@ -189,7 +189,6 @@ export class CallEventHandler {
const groupCallId = content.conf_id; const groupCallId = content.conf_id;
const type = event.getType() as EventType; const type = event.getType() as EventType;
const senderId = event.getSender()!; const senderId = event.getSender()!;
const weSentTheEvent = senderId === this.client.credentials.userId;
let call = content.call_id ? this.calls.get(content.call_id) : undefined; let call = content.call_id ? this.calls.get(content.call_id) : undefined;
let opponentDeviceId: string | undefined; let opponentDeviceId: string | undefined;
@ -220,6 +219,9 @@ export class CallEventHandler {
} }
} }
const weSentTheEvent = senderId === this.client.credentials.userId
&& (opponentDeviceId === undefined || opponentDeviceId === this.client.getDeviceId()!);
if (!callRoomId) return; if (!callRoomId) return;
if (type === EventType.CallInvite) { if (type === EventType.CallInvite) {

7
src/webrtc/callFeed.ts
View File

@ -29,6 +29,7 @@ export interface ICallFeedOpts {
client: MatrixClient; client: MatrixClient;
roomId?: string; roomId?: string;
userId: string; userId: string;
deviceId: string | undefined;
stream: MediaStream; stream: MediaStream;
purpose: SDPStreamMetadataPurpose; purpose: SDPStreamMetadataPurpose;
/** /**
@ -63,6 +64,7 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
public stream: MediaStream; public stream: MediaStream;
public sdpMetadataStreamId: string; public sdpMetadataStreamId: string;
public userId: string; public userId: string;
public readonly deviceId: string | undefined;
public purpose: SDPStreamMetadataPurpose; public purpose: SDPStreamMetadataPurpose;
public speakingVolumeSamples: number[]; public speakingVolumeSamples: number[];
@ -86,6 +88,7 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
this.client = opts.client; this.client = opts.client;
this.roomId = opts.roomId; this.roomId = opts.roomId;
this.userId = opts.userId; this.userId = opts.userId;
this.deviceId = opts.deviceId;
this.purpose = opts.purpose; this.purpose = opts.purpose;
this.audioMuted = opts.audioMuted; this.audioMuted = opts.audioMuted;
this.videoMuted = opts.videoMuted; this.videoMuted = opts.videoMuted;
@ -156,7 +159,8 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
* @returns {boolean} is local? * @returns {boolean} is local?
*/ */
public isLocal(): boolean { public isLocal(): boolean {
return this.userId === this.client.getUserId(); return this.userId === this.client.getUserId()
&& (this.deviceId === undefined || this.deviceId === this.client.getDeviceId());
} }
/** /**
@ -282,6 +286,7 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
client: this.client, client: this.client,
roomId: this.roomId, roomId: this.roomId,
userId: this.userId, userId: this.userId,
deviceId: this.deviceId,
stream, stream,
purpose: this.purpose, purpose: this.purpose,
audioMuted: this.audioMuted, audioMuted: this.audioMuted,

1068
src/webrtc/groupCall.ts
View File

File diff suppressed because it is too large Load Diff

8
src/webrtc/groupCallEventHandler.ts
View File

@ -220,14 +220,6 @@ export class GroupCallEventHandler {
logger.warn(`Multiple group calls detected for room: ${ logger.warn(`Multiple group calls detected for room: ${
state.roomId}. Multiple group calls are currently unsupported.`); state.roomId}. Multiple group calls are currently unsupported.`);
} }
} else if (eventType === EventType.GroupCallMemberPrefix) {
const groupCall = this.groupCalls.get(state.roomId);
if (!groupCall) {
return;
}
groupCall.onMemberStateChanged(event);
} }
}; };
} }