matrix/matrix-js-sdk
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-js-sdk.git synced 2025-08-07 23:02:56 +03:00
Code Activity

Refactor GroupCall participant management

Browse Source 
This refactoring brings a number of improvements to GroupCall, which I've unfortunately had to combine into a single commit due to coupling:

- Moves the expiration timestamp field on call membership state to be per-device
- Makes the participants of a group call visible without having to enter the call yourself
- Enables users to join group calls from multiple devices
- Identifies active speakers by their call feed, rather than just their user ID
- Plays nicely with clients that can be in multiple calls in a room at once
- Fixes a memory leak caused by the call retry loop never stopping
- Changes GroupCall to update its state synchronously, and write back to room state asynchronously
  - This was already sort of halfway being done, but now we'd be committing to it
  - Generally improves the robustness of the state machine
  - It means that group call joins will appear instant, in a sense

For many reasons, this is a breaking change.
This commit is contained in:
Robin Townsend
2022-11-21 11:58:27 -05:00
parent dd98d7eb2c
commit f46ecf970c
10 changed files with 735 additions and 668 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
spec/unit/webrtc/groupCall.spec.ts
View File

@@ -23,6 +23,7 @@ import {
Room,
RoomMember,
} from '../../../src';
import { RoomStateEvent } from "../../../src/models/room-state";
import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall";
import { MatrixClient } from "../../../src/client";
import {
@@ -53,18 +54,19 @@ const FAKE_USER_ID_3 = "@charlie:test.dummy";
const FAKE_STATE_EVENTS = [
{
getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR,
"m.calls": [],
}),
getStateKey: () => FAKE_USER_ID_1,
getRoomId: () => FAKE_ROOM_ID,
},
{
getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR,
["m.calls"]: [{
["m.call_id"]: FAKE_CONF_ID,
["m.devices"]: [{
"m.calls": [{
"m.call_id": FAKE_CONF_ID,
"m.devices": [{
device_id: FAKE_DEVICE_ID_2,
session_id: FAKE_SESSION_ID_2,
expires_ts: Date.now() + ONE_HOUR,
feeds: [],
}],
}],
@@ -73,11 +75,13 @@ const FAKE_STATE_EVENTS = [
getRoomId: () => FAKE_ROOM_ID,
}, {
getContent: () => ({
["m.expires_ts"]: Date.now() + ONE_HOUR,
["m.calls"]: [{
["m.call_id"]: FAKE_CONF_ID,
["m.devices"]: [{
"m.expires_ts": Date.now() + ONE_HOUR,
"m.calls": [{
"m.call_id": FAKE_CONF_ID,
"m.devices": [{
device_id: "user3_device",
session_id: "user3_session",
expires_ts: Date.now() + ONE_HOUR,
feeds: [],
}],
}],
@@ -111,6 +115,8 @@ class MockCall {
public state = CallState.Ringing;
public opponentUserId = FAKE_USER_ID_1;
public opponentDeviceId = FAKE_DEVICE_ID_1;
public opponentMember = { userId: this.opponentUserId };
public callId = "1";
public localUsermediaFeed = {
setAudioVideoMuted: jest.fn<void, [boolean, boolean]>(),
@@ -129,9 +135,11 @@ class MockCall {
public removeListener = jest.fn();
public getOpponentMember(): Partial<RoomMember> {
return {
userId: this.opponentUserId,
};
return this.opponentMember;
}
public getOpponentDeviceId(): string {
return this.opponentDeviceId;
}
public typed(): MatrixCall { return this as unknown as MatrixCall; }
@@ -326,8 +334,8 @@ describe('Group Call', function() {
describe("call feeds changing", () => {
let call: MockCall;
const currentFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("current"));
const newFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("new"));
const currentFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("current"));
const newFeed = new MockCallFeed(FAKE_USER_ID_1, FAKE_DEVICE_ID_1, new MockMediaStream("new"));
beforeEach(async () => {
jest.spyOn(currentFeed, "dispose");
@@ -358,7 +366,7 @@ describe('Group Call', function() {
});
it("replaces usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()];
groupCall.userMediaFeeds.push(currentFeed.typed());
call.remoteUsermediaFeed = newFeed.typed();
// @ts-ignore Mock
@@ -368,7 +376,7 @@ describe('Group Call', function() {
});
it("removes usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()];
groupCall.userMediaFeeds.push(currentFeed.typed());
// @ts-ignore Mock
groupCall.onCallFeedsChanged(call);
@@ -387,7 +395,7 @@ describe('Group Call', function() {
});
it("replaces screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()];
groupCall.screenshareFeeds.push(currentFeed.typed());
call.remoteScreensharingFeed = newFeed.typed();
// @ts-ignore Mock
@@ -397,7 +405,7 @@ describe('Group Call', function() {
});
it("removes screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()];
groupCall.screenshareFeeds.push(currentFeed.typed());
// @ts-ignore Mock
groupCall.onCallFeedsChanged(call);
@@ -408,7 +416,7 @@ describe('Group Call', function() {
describe("feed replacing", () => {
it("replaces usermedia feed", async () => {
groupCall.userMediaFeeds = [currentFeed.typed()];
groupCall.userMediaFeeds.push(currentFeed.typed());
// @ts-ignore Mock
groupCall.replaceUserMediaFeed(currentFeed, newFeed);
@@ -422,7 +430,7 @@ describe('Group Call', function() {
});
it("replaces screenshare feed", async () => {
groupCall.screenshareFeeds = [currentFeed.typed()];
groupCall.screenshareFeeds.push(currentFeed.typed());
// @ts-ignore Mock
groupCall.replaceScreenshareFeed(currentFeed, newFeed);
@@ -489,7 +497,10 @@ describe('Group Call', function() {
it("sends metadata updates before unmuting in PTT mode", async () => {
const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId);
groupCall.calls.push(mockCall as unknown as MatrixCall);
groupCall.calls.set(
mockCall.getOpponentMember() as RoomMember,
new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]),
);
let metadataUpdateResolve: () => void;
const metadataUpdatePromise = new Promise<void>(resolve => {
@@ -511,7 +522,10 @@ describe('Group Call', function() {
it("sends metadata updates after muting in PTT mode", async () => {
const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId);
groupCall.calls.push(mockCall as unknown as MatrixCall);
groupCall.calls.set(
mockCall.getOpponentMember() as RoomMember,
new Map([[mockCall.getOpponentDeviceId(), mockCall.typed()]]),
);
// the call starts muted, so unmute to get in the right state to test
await groupCall.setMicrophoneMuted(false);
@@ -560,7 +574,7 @@ describe('Group Call', function() {
if (eventType === EventType.GroupCallMemberPrefix) {
const fakeEvent = {
getContent: () => content,
getRoomId: () => FAKE_ROOM_ID,
getRoomId: () => roomId,
getStateKey: () => statekey,
} as unknown as MatrixEvent;
@@ -574,8 +588,8 @@ describe('Group Call', function() {
// just add it once.
subMap.set(statekey, fakeEvent);
groupCall1.onMemberStateChanged(fakeEvent);
groupCall2.onMemberStateChanged(fakeEvent);
client1Room.currentState.emit(RoomStateEvent.Update, client1Room.currentState);
client2Room.currentState.emit(RoomStateEvent.Update, client2Room.currentState);
}
return Promise.resolve({ "event_id": "foo" });
};
@@ -584,9 +598,17 @@ describe('Group Call', function() {
client2.sendStateEvent.mockImplementation(fakeSendStateEvents);
const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1);
const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2);
client1Room.currentState.members[FAKE_USER_ID_1] = client2Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
client1Room.currentState.members[FAKE_USER_ID_2] = client2Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
groupCall1 = new GroupCall(
client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID,
);
@@ -594,20 +616,6 @@ describe('Group Call', function() {
groupCall2 = new GroupCall(
client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID,
);
client1Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
client1Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
} as unknown as RoomMember;
client2Room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
client2Room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
} as unknown as RoomMember;
});
afterEach(function() {
@@ -672,8 +680,10 @@ describe('Group Call', function() {
expect(client1.sendToDevice).toHaveBeenCalled();
const oldCall = groupCall1.getCallByUserId(client2.userId);
oldCall!.emit(CallEvent.Hangup, oldCall!);
const oldCall = groupCall1.calls.get(
groupCall1.room.getMember(client2.userId)!,
)!.get(client2.deviceId)!;
oldCall.emit(CallEvent.Hangup, oldCall!);
client1.sendToDevice.mockClear();
@@ -691,9 +701,11 @@ describe('Group Call', function() {
// to even be created...
let newCall: MatrixCall | undefined;
while (
(newCall = groupCall1.getCallByUserId(client2.userId)) === undefined ||
newCall.peerConn === undefined ||
newCall.callId == oldCall!.callId
(newCall = groupCall1.calls.get(
groupCall1.room.getMember(client2.userId)!,
)?.get(client2.deviceId)) === undefined
|| newCall.peerConn === undefined
|| newCall.callId == oldCall.callId
) {
await flushPromises();
}
@@ -733,7 +745,9 @@ describe('Group Call', function() {
groupCall1.setMicrophoneMuted(false);
groupCall1.setLocalVideoMuted(false);
const call = groupCall1.getCallByUserId(client2.userId)!;
const call = groupCall1.calls.get(
groupCall1.room.getMember(client2.userId)!,
)!.get(client2.deviceId)!;
call.isMicrophoneMuted = jest.fn().mockReturnValue(true);
call.setMicrophoneMuted = jest.fn();
call.isLocalVideoMuted = jest.fn().mockReturnValue(true);
@@ -765,7 +779,14 @@ describe('Group Call', function() {
? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS
: { getContent: () => ([]) };
});
room.getMember = jest.fn().mockImplementation((userId) => ({ userId }));
room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
});
describe("local muting", () => {
@@ -773,17 +794,13 @@ describe('Group Call', function() {
const groupCall = await createAndEnterGroupCall(mockClient, room);
groupCall.localCallFeed!.setAudioVideoMuted = jest.fn();
const setAVMutedArray = groupCall.calls.map(call => {
call.localUsermediaFeed!.setAudioVideoMuted = jest.fn();
return call.localUsermediaFeed!.setAudioVideoMuted;
});
const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => {
acc.push(...call.localUsermediaStream!.getAudioTracks());
return acc;
}, []);
const sendMetadataUpdateArray = groupCall.calls.map(call => {
call.sendMetadataUpdate = jest.fn();
return call.sendMetadataUpdate;
const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = [];
const tracksArray: MediaStreamTrack[] = [];
const sendMetadataUpdateArray: (() => Promise<void>)[] = [];
groupCall.forEachCall(call => {
setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn());
tracksArray.push(...call.localUsermediaStream!.getAudioTracks());
sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn());
});
await groupCall.setMicrophoneMuted(true);
@@ -801,18 +818,14 @@ describe('Group Call', function() {
const groupCall = await createAndEnterGroupCall(mockClient, room);
groupCall.localCallFeed!.setAudioVideoMuted = jest.fn();
const setAVMutedArray = groupCall.calls.map(call => {
call.localUsermediaFeed!.setAudioVideoMuted = jest.fn();
const setAVMutedArray: ((audioMuted: boolean | null, videoMuted: boolean | null) => void)[] = [];
const tracksArray: MediaStreamTrack[] = [];
const sendMetadataUpdateArray: (() => Promise<void>)[] = [];
groupCall.forEachCall(call => {
call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true);
return call.localUsermediaFeed!.setAudioVideoMuted;
});
const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => {
acc.push(...call.localUsermediaStream!.getVideoTracks());
return acc;
}, []);
const sendMetadataUpdateArray = groupCall.calls.map(call => {
call.sendMetadataUpdate = jest.fn();
return call.sendMetadataUpdate;
setAVMutedArray.push(call.localUsermediaFeed!.setAudioVideoMuted = jest.fn());
tracksArray.push(...call.localUsermediaStream!.getVideoTracks());
sendMetadataUpdateArray.push(call.sendMetadataUpdate = jest.fn());
});
await groupCall.setLocalVideoMuted(true);
@@ -847,7 +860,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created
await sleep(10);
const call = groupCall.calls[0];
const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
// @ts-ignore Mock
call.pushRemoteFeed(new MockMediaStream("stream", [
@@ -856,7 +869,7 @@ describe('Group Call', function() {
]));
call.onSDPStreamMetadataChangedReceived(metadataEvent);
const feed = groupCall.getUserMediaFeedByUserId(call.invitee!);
const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!);
expect(feed!.isAudioMuted()).toBe(true);
expect(feed!.isVideoMuted()).toBe(false);
@@ -870,7 +883,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created
await sleep(10);
const call = groupCall.calls[0];
const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
// @ts-ignore Mock
call.pushRemoteFeed(new MockMediaStream("stream", [
@@ -879,7 +892,7 @@ describe('Group Call', function() {
]));
call.onSDPStreamMetadataChangedReceived(metadataEvent);
const feed = groupCall.getUserMediaFeedByUserId(call.invitee!);
const feed = groupCall.getUserMediaFeed(call.invitee!, call.getOpponentDeviceId()!);
expect(feed!.isAudioMuted()).toBe(false);
expect(feed!.isVideoMuted()).toBe(true);
@@ -945,12 +958,16 @@ describe('Group Call', function() {
expect(mockCall.reject).not.toHaveBeenCalled();
expect(mockCall.answerWithCallFeeds).toHaveBeenCalled();
expect(groupCall.calls).toEqual([mockCall]);
expect(groupCall.calls).toEqual(new Map([[
groupCall.room.getMember(FAKE_USER_ID_1)!,
new Map([[FAKE_DEVICE_ID_1, mockCall]]),
]]));
});
it("replaces calls if it already has one with the same user", async () => {
const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId);
const newMockCall = new MockCall(room.roomId, groupCall.groupCallId);
newMockCall.opponentMember = oldMockCall.opponentMember; // Ensure referential equality
newMockCall.callId = "not " + oldMockCall.callId;
mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall);
@@ -958,7 +975,10 @@ describe('Group Call', function() {
expect(oldMockCall.hangup).toHaveBeenCalled();
expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled();
expect(groupCall.calls).toEqual([newMockCall]);
expect(groupCall.calls).toEqual(new Map([[
groupCall.room.getMember(FAKE_USER_ID_1)!,
new Map([[FAKE_DEVICE_ID_1, newMockCall]]),
]]));
});
it("starts to process incoming calls when we've entered", async () => {
@@ -988,7 +1008,14 @@ describe('Group Call', function() {
mockClient = typedMockClient.typed();
room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1);
room.getMember = jest.fn().mockImplementation((userId) => ({ userId }));
room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
membership: "join",
} as unknown as RoomMember;
room.currentState.members[FAKE_USER_ID_2] = {
userId: FAKE_USER_ID_2,
membership: "join",
} as unknown as RoomMember;
room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => {
return type === EventType.GroupCallMemberPrefix
? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS
@@ -999,21 +1026,20 @@ describe('Group Call', function() {
});
it("sending screensharing stream", async () => {
const onNegotiationNeededArray = groupCall.calls.map(call => {
const onNegotiationNeededArray: (() => Promise<void>)[] = [];
groupCall.forEachCall(call => {
// @ts-ignore Mock
call.gotLocalOffer = jest.fn();
// @ts-ignore Mock
return call.gotLocalOffer;
onNegotiationNeededArray.push(call.gotLocalOffer = jest.fn());
});
let enabledResult;
let enabledResult: boolean;
enabledResult = await groupCall.setScreensharingEnabled(true);
expect(enabledResult).toEqual(true);
expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled();
MockRTCPeerConnection.triggerAllNegotiations();
expect(groupCall.screenshareFeeds).toHaveLength(1);
groupCall.calls.forEach(c => {
groupCall.forEachCall(c => {
expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined();
});
onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled());
@@ -1036,7 +1062,7 @@ describe('Group Call', function() {
// It takes a bit of time for the calls to get created
await sleep(10);
const call = groupCall.calls[0];
const call = groupCall.calls.get(groupCall.room.getMember(FAKE_USER_ID_2)!)!.get(FAKE_DEVICE_ID_2)!;
call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember;
call.onNegotiateReceived({
getContent: () => ({
@@ -1057,7 +1083,7 @@ describe('Group Call', function() {
]));
expect(groupCall.screenshareFeeds).toHaveLength(1);
expect(groupCall.getScreenshareFeedByUserId(call.invitee!)).toBeDefined();
expect(groupCall.getScreenshareFeed(call.invitee!, call.getOpponentDeviceId()!)).toBeDefined();
groupCall.terminate();
});
@@ -1097,12 +1123,16 @@ describe('Group Call', function() {
);
room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1);
room.currentState.members[FAKE_USER_ID_1] = {
userId: FAKE_USER_ID_1,
} as unknown as RoomMember;
groupCall = await createAndEnterGroupCall(mockClient.typed(), room);
mediaFeed1 = new CallFeed({
client: mockClient.typed(),
roomId: FAKE_ROOM_ID,
userId: FAKE_USER_ID_2,
deviceId: FAKE_DEVICE_ID_1,
stream: (new MockMediaStream("foo", [])).typed(),
purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false,
@@ -1114,6 +1144,7 @@ describe('Group Call', function() {
client: mockClient.typed(),
roomId: FAKE_ROOM_ID,
userId: FAKE_USER_ID_3,
deviceId: FAKE_DEVICE_ID_1,
stream: (new MockMediaStream("foo", [])).typed(),
purpose: SDPStreamMetadataPurpose.Usermedia,
audioMuted: false,
@@ -1136,15 +1167,15 @@ describe('Group Call', function() {
mediaFeed2.speakingVolumeSamples = [0, 0];
jest.runOnlyPendingTimers();
expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_2);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_2);
expect(groupCall.activeSpeaker).toEqual(mediaFeed1);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed1);
mediaFeed1.speakingVolumeSamples = [0, 0];
mediaFeed2.speakingVolumeSamples = [100, 100];
jest.runOnlyPendingTimers();
expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_3);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3);
expect(groupCall.activeSpeaker).toEqual(mediaFeed2);
expect(onActiveSpeakerEvent).toHaveBeenCalledWith(mediaFeed2);
});
});