diff --git a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts index 813b5d139..70cbe927d 100644 --- a/spec/unit/matrixrtc/MatrixRTCSession.spec.ts +++ b/spec/unit/matrixrtc/MatrixRTCSession.spec.ts @@ -222,6 +222,27 @@ describe("MatrixRTCSession", () => { }); }); + describe("getConsensusCallIntent", () => { + it.each([ + [undefined, undefined, undefined], + ["audio", undefined, "audio"], + [undefined, "audio", "audio"], + ["audio", "audio", "audio"], + ["audio", "video", undefined], + ])("gets correct consensus for %s + %s = %s", (intentA, intentB, result) => { + jest.useFakeTimers(); + jest.setSystemTime(4000); + const mockRoom = makeMockRoom([ + Object.assign({}, membershipTemplate, { "m.call.intent": intentA }), + Object.assign({}, membershipTemplate, { "m.call.intent": intentB }), + ]); + + sess = MatrixRTCSession.sessionForRoom(client, mockRoom, callSession); + expect(sess.getConsensusCallIntent()).toEqual(result); + jest.useRealTimers(); + }); + }); + describe("getsActiveFocus", () => { const firstPreferredFocus = { type: "livekit", @@ -370,6 +391,79 @@ describe("MatrixRTCSession", () => { ); }); + it("sends a notification with a intent when starting a call and emits DidSendCallNotification", async () => { + // Simulate a join, including the update to the room state + // Ensure sendEvent returns event IDs so the DidSendCallNotification payload includes them + sendEventMock + .mockResolvedValueOnce({ event_id: "legacy-evt" }) + .mockResolvedValueOnce({ event_id: "new-evt" }); + const didSendEventFn = jest.fn(); + sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, didSendEventFn); + // Create an additional listener to create a promise that resolves after the emission. + const didSendNotification = new Promise((resolve) => { + sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, resolve); + }); + + sess!.joinRoomSession([mockFocus], mockFocus, { notificationType: "ring", callIntent: "audio" }); + await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]); + + mockRoomState(mockRoom, [ + { + ...membershipTemplate, + "user_id": client.getUserId()!, + // This is what triggers the intent type on the notification event. + "m.call.intent": "audio", + }, + ]); + + sess!.onRTCSessionMemberUpdate(); + const ownMembershipId = sess?.memberships[0].eventId; + expect(sess!.getConsensusCallIntent()).toEqual("audio"); + + expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.RTCNotification, { + "m.mentions": { user_ids: [], room: true }, + "notification_type": "ring", + "m.call.intent": "audio", + "m.relates_to": { + event_id: ownMembershipId, + rel_type: "m.reference", + }, + "lifetime": 30000, + "sender_ts": expect.any(Number), + }); + + // Check if deprecated notify event is also sent. + expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.CallNotify, { + "application": "m.call", + "m.mentions": { user_ids: [], room: true }, + "notify_type": "ring", + "call_id": "", + }); + await didSendNotification; + // And ensure we emitted the DidSendCallNotification event with both payloads + expect(didSendEventFn).toHaveBeenCalledWith( + { + "event_id": "new-evt", + "lifetime": 30000, + "m.mentions": { room: true, user_ids: [] }, + "m.relates_to": { + event_id: expect.any(String), + rel_type: "m.reference", + }, + "notification_type": "ring", + "m.call.intent": "audio", + "sender_ts": expect.any(Number), + }, + { + "application": "m.call", + "call_id": "", + "event_id": "legacy-evt", + "m.mentions": { room: true, user_ids: [] }, + "notify_type": "ring", + }, + ); + }); + it("doesn't send a notification when joining an existing call", async () => { // Add another member to the call so that it is considered an existing call mockRoomState(mockRoom, [membershipTemplate]); diff --git a/spec/unit/matrixrtc/MembershipManager.spec.ts b/spec/unit/matrixrtc/MembershipManager.spec.ts index bf42fad93..c22ab1839 100644 --- a/spec/unit/matrixrtc/MembershipManager.spec.ts +++ b/spec/unit/matrixrtc/MembershipManager.spec.ts @@ -901,6 +901,43 @@ describe("MembershipManager", () => { } }); }); + + describe("updateCallIntent()", () => { + it("should fail if the user has not joined the call", async () => { + const manager = new MembershipManager({}, room, client, () => undefined, callSession); + // After joining we want our own focus to be the one we select. + try { + await manager.updateCallIntent("video"); + throw Error("Should have thrown"); + } catch {} + }); + + it("can adjust the intent", async () => { + const manager = new MembershipManager({}, room, client, () => undefined, callSession); + manager.join([]); + expect(manager.isActivated()).toEqual(true); + const membership = mockCallMembership({ ...membershipTemplate, user_id: client.getUserId()! }, room.roomId); + await manager.onRTCSessionMemberUpdate([membership]); + await manager.updateCallIntent("video"); + expect(client.sendStateEvent).toHaveBeenCalledTimes(2); + const eventContent = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData; + expect(eventContent["created_ts"]).toEqual(membership.createdTs()); + expect(eventContent["m.call.intent"]).toEqual("video"); + }); + + it("does nothing if the intent doesn't change", async () => { + const manager = new MembershipManager({ callIntent: "video" }, room, client, () => undefined, callSession); + manager.join([]); + expect(manager.isActivated()).toEqual(true); + const membership = mockCallMembership( + { ...membershipTemplate, "user_id": client.getUserId()!, "m.call.intent": "video" }, + room.roomId, + ); + await manager.onRTCSessionMemberUpdate([membership]); + await manager.updateCallIntent("video"); + expect(client.sendStateEvent).toHaveBeenCalledTimes(0); + }); + }); }); it("Should prefix log with MembershipManager used", () => { diff --git a/src/matrixrtc/CallMembership.ts b/src/matrixrtc/CallMembership.ts index b3f50f028..00e42baa1 100644 --- a/src/matrixrtc/CallMembership.ts +++ b/src/matrixrtc/CallMembership.ts @@ -19,6 +19,7 @@ import { deepCompare } from "../utils.ts"; import { type Focus } from "./focus.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { type SessionDescription } from "./MatrixRTCSession.ts"; +import { type RTCCallIntent } from "./types.ts"; /** * The default duration in milliseconds that a membership is considered valid for. @@ -31,13 +32,13 @@ type CallScope = "m.room" | "m.user"; /** * MSC4143 (MatrixRTC) session membership data. - * Represents an entry in the memberships section of an m.call.member event as it is on the wire. + * Represents the `session` in the memberships section of an m.call.member event as it is on the wire. **/ export type SessionMembershipData = { /** * The RTC application defines the type of the RTC session. */ - application: string; + "application": string; /** * The id of this session. @@ -45,23 +46,23 @@ export type SessionMembershipData = { * multiple session in one room. A room wide session that is not associated with a user, * and therefore immune to creation race conflicts, uses the `call_id: ""`. */ - call_id: string; + "call_id": string; /** * The Matrix device ID of this session. A single user can have multiple sessions on different devices. */ - device_id: string; + "device_id": string; /** * The focus selection system this user/membership is using. */ - focus_active: Focus; + "focus_active": Focus; /** * A list of possible foci this uses knows about. One of them might be used based on the focus_active * selection system. */ - foci_preferred: Focus[]; + "foci_preferred": Focus[]; /** * Optional field that contains the creation of the session. If it is undefined the creation @@ -70,7 +71,7 @@ export type SessionMembershipData = { * - If it is undefined it can be interpreted as a "Join". * - If it is defined it can be interpreted as an "Update" */ - created_ts?: number; + "created_ts"?: number; // Application specific data @@ -78,17 +79,26 @@ export type SessionMembershipData = { * If the `application` = `"m.call"` this defines if it is a room or user owned call. * There can always be one room scroped call but multiple user owned calls (breakout sessions) */ - scope?: CallScope; + "scope"?: CallScope; /** * Optionally we allow to define a delta to the `created_ts` that defines when the event is expired/invalid. * This should be set to multiple hours. The only reason it exist is to deal with failed delayed events. * (for example caused by a homeserver crashes) **/ - expires?: number; + "expires"?: number; + + /** + * The intent of the call from the perspective of this user. This may be an audio call, video call or + * something else. + */ + "m.call.intent"?: RTCCallIntent; }; -const checkSessionsMembershipData = (data: any, errors: string[]): data is SessionMembershipData => { +const checkSessionsMembershipData = ( + data: Partial>, + errors: string[], +): data is SessionMembershipData => { const prefix = "Malformed session membership event: "; if (typeof data.device_id !== "string") errors.push(prefix + "device_id must be string"); if (typeof data.call_id !== "string") errors.push(prefix + "call_id must be string"); @@ -96,10 +106,17 @@ const checkSessionsMembershipData = (data: any, errors: string[]): data is Sessi if (typeof data.focus_active?.type !== "string") errors.push(prefix + "focus_active.type must be a string"); if (!Array.isArray(data.foci_preferred)) errors.push(prefix + "foci_preferred must be an array"); // optional parameters - if (data.created_ts && typeof data.created_ts !== "number") errors.push(prefix + "created_ts must be number"); + if (data.created_ts !== undefined && typeof data.created_ts !== "number") { + errors.push(prefix + "created_ts must be number"); + } // application specific data (we first need to check if they exist) - if (data.scope && typeof data.scope !== "string") errors.push(prefix + "scope must be string"); + if (data.scope !== undefined && typeof data.scope !== "string") errors.push(prefix + "scope must be string"); + + if (data["m.call.intent"] !== undefined && typeof data["m.call.intent"] !== "string") { + errors.push(prefix + "m.call.intent must be a string"); + } + return errors.length === 0; }; @@ -142,6 +159,10 @@ export class CallMembership { return this.membershipData.device_id; } + public get callIntent(): RTCCallIntent | undefined { + return this.membershipData["m.call.intent"]; + } + public get sessionDescription(): SessionDescription { return { application: this.membershipData.application, diff --git a/src/matrixrtc/IMembershipManager.ts b/src/matrixrtc/IMembershipManager.ts index b1702ef8e..320654543 100644 --- a/src/matrixrtc/IMembershipManager.ts +++ b/src/matrixrtc/IMembershipManager.ts @@ -16,7 +16,7 @@ limitations under the License. import type { CallMembership } from "./CallMembership.ts"; import type { Focus } from "./focus.ts"; -import type { Status } from "./types.ts"; +import type { RTCCallIntent, Status } from "./types.ts"; import { type TypedEventEmitter } from "../models/typed-event-emitter.ts"; export enum MembershipManagerEvent { @@ -100,4 +100,10 @@ export interface IMembershipManager * @returns the used active focus in the currently joined session or undefined if not joined. */ getActiveFocus(): Focus | undefined; + + /** + * Update the intent of a membership on the call (e.g. user is now providing a video feed) + * @param callIntent The new intent to set. + */ + updateCallIntent(callIntent: RTCCallIntent): Promise; } diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index b8c5c2940..020f76855 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -34,6 +34,7 @@ import { type Status, type IRTCNotificationContent, type ICallNotifyContent, + type RTCCallIntent, } from "./types.ts"; import { RoomKeyTransport } from "./RoomKeyTransport.ts"; import { @@ -92,6 +93,11 @@ export interface SessionConfig { * @default `undefined` (no notification) */ notificationType?: RTCNotificationType; + + /** + * Determines the kind of call this will be. + */ + callIntent?: RTCCallIntent; } /** @@ -614,6 +620,32 @@ export class MatrixRTCSession extends TypedEventEmitter< return this.memberships[0]; } + /** + * Get the call intent for the current call, based on what members are advertising. If one or more + * members disagree on the current call intent, or nobody specifies one then `undefined` is returned. + * + * If all members that specify a call intent agree, that value is returned. + * @returns A call intent, or `undefined` if no consensus or not given. + */ + public getConsensusCallIntent(): RTCCallIntent | undefined { + const getFirstCallIntent = this.memberships.find((m) => !!m.callIntent)?.callIntent; + if (!getFirstCallIntent) { + return undefined; + } + if (this.memberships.every((m) => !m.callIntent || m.callIntent === getFirstCallIntent)) { + return getFirstCallIntent; + } + return undefined; + } + + public async updateCallIntent(callIntent: RTCCallIntent): Promise { + const myMembership = this.membershipManager?.ownMembership; + if (!myMembership) { + throw Error("Not connected yet"); + } + await this.membershipManager?.updateCallIntent(callIntent); + } + /** * This method is used when the user is not yet connected to the Session but wants to know what focus * the users in the session are using to make a decision how it wants/should connect. @@ -665,9 +697,17 @@ export class MatrixRTCSession extends TypedEventEmitter< } /** - * Sends a notification corresponding to the configured notify type. + * Sends notification events to indiciate the call has started. + * Note: This does not return a promise, instead scheduling the notification events to be sent. + * @param parentEventId Event id linking to your RTC call membership event. + * @param notificationType The type of notification to send + * @param callIntent The type of call this is (e.g. "audio"). */ - private sendCallNotify(parentEventId: string, notificationType: RTCNotificationType): void { + private sendCallNotify( + parentEventId: string, + notificationType: RTCNotificationType, + callIntent?: RTCCallIntent, + ): void { const sendLegacyNotificationEvent = async (): Promise<{ response: ISendEventResponse; content: ICallNotifyContent; @@ -695,6 +735,9 @@ export class MatrixRTCSession extends TypedEventEmitter< "sender_ts": Date.now(), "lifetime": 30_000, // 30 seconds }; + if (callIntent) { + content["m.call.intent"] = callIntent; + } const response = await this.client.sendEvent(this.roomSubset.roomId, EventType.RTCNotification, content); return { response, content }; }; @@ -757,7 +800,11 @@ export class MatrixRTCSession extends TypedEventEmitter< // If we're the first member in the call, we're responsible for // sending the notification event if (ownMembership.eventId && this.joinConfig?.notificationType) { - this.sendCallNotify(ownMembership.eventId, this.joinConfig.notificationType); + this.sendCallNotify( + ownMembership.eventId, + this.joinConfig.notificationType, + ownMembership.callIntent, + ); } else { this.logger.warn("Own membership eventId is undefined, cannot send call notification"); } diff --git a/src/matrixrtc/MembershipManager.ts b/src/matrixrtc/MembershipManager.ts index cdee3ee83..9e50e92d4 100644 --- a/src/matrixrtc/MembershipManager.ts +++ b/src/matrixrtc/MembershipManager.ts @@ -24,9 +24,9 @@ import { type Logger, logger as rootLogger } from "../logger.ts"; import { type Room } from "../models/room.ts"; import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts"; import { type Focus } from "./focus.ts"; -import { isMyMembership, Status } from "./types.ts"; +import { isMyMembership, type RTCCallIntent, Status } from "./types.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; -import { type SessionDescription, type MembershipConfig } from "./MatrixRTCSession.ts"; +import { type SessionDescription, type MembershipConfig, type SessionConfig } from "./MatrixRTCSession.ts"; import { ActionScheduler, type ActionUpdate } from "./MembershipManagerActionScheduler.ts"; import { TypedEventEmitter } from "../models/typed-event-emitter.ts"; import { @@ -156,6 +156,7 @@ export class MembershipManager { private activated = false; private logger: Logger; + private callIntent: RTCCallIntent | undefined; public isActivated(): boolean { return this.activated; @@ -230,7 +231,10 @@ export class MembershipManager private leavePromiseResolvers?: PromiseWithResolvers; - public async onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { + public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise { + if (!this.isActivated()) { + return Promise.resolve(); + } const userId = this.client.getUserId(); const deviceId = this.client.getDeviceId(); if (!userId || !deviceId) { @@ -239,7 +243,7 @@ export class MembershipManager } this._ownMembership = memberships.find((m) => isMyMembership(m, userId, deviceId)); - if (this.isActivated() && !this._ownMembership) { + if (!this._ownMembership) { // If one of these actions are scheduled or are getting inserted in the next iteration, we should already // take care of our missing membership. const sendingMembershipActions = [ @@ -281,6 +285,18 @@ export class MembershipManager } } + public async updateCallIntent(callIntent: RTCCallIntent): Promise { + if (!this.activated || !this.ownMembership) { + throw Error("You cannot update your intent before joining the call"); + } + if (this.ownMembership.callIntent === callIntent) { + return; // No-op + } + this.callIntent = callIntent; + // Kick off a new membership event as a result. + await this.sendJoinEvent(); + } + /** * @throws if the client does not return user or device id. * @param joinConfig @@ -289,7 +305,7 @@ export class MembershipManager * @param getOldestMembership */ public constructor( - private joinConfig: MembershipConfig | undefined, + private joinConfig: (SessionConfig & MembershipConfig) | undefined, private room: Pick, private client: Pick< MatrixClient, @@ -311,6 +327,7 @@ export class MembershipManager this.deviceId = deviceId; this.stateKey = this.makeMembershipStateKey(userId, deviceId); this.state = MembershipManager.defaultState; + this.callIntent = joinConfig?.callIntent; this.scheduler = new ActionScheduler((type): Promise => { if (this.oldStatus) { // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this @@ -741,15 +758,18 @@ export class MembershipManager * Constructs our own membership */ private makeMyMembership(expires: number): SessionMembershipData { + const hasPreviousEvent = !!this.ownMembership; return { // TODO: use the new format for m.rtc.member events where call_id becomes session.id - application: this.sessionDescription.application, - call_id: this.sessionDescription.id, - scope: "m.room", - device_id: this.deviceId, + "application": this.sessionDescription.application, + "call_id": this.sessionDescription.id, + "scope": "m.room", + "device_id": this.deviceId, expires, - focus_active: { type: "livekit", focus_selection: "oldest_membership" }, - foci_preferred: this.fociPreferred ?? [], + "focus_active": { type: "livekit", focus_selection: "oldest_membership" }, + "foci_preferred": this.fociPreferred ?? [], + "m.call.intent": this.callIntent, + ...(hasPreviousEvent ? { created_ts: this.ownMembership?.createdTs() } : undefined), }; } diff --git a/src/matrixrtc/types.ts b/src/matrixrtc/types.ts index 1f7bbe3a6..b344a22d8 100644 --- a/src/matrixrtc/types.ts +++ b/src/matrixrtc/types.ts @@ -96,10 +96,20 @@ export interface ICallNotifyContent { } export type RTCNotificationType = "ring" | "notification"; + +/** + * Represents the intention of the call from the perspective of the sending user. + * May be any string, although `"audio"` and `"video"` are commonly accepted values. + */ +export type RTCCallIntent = "audio" | "video" | string; export interface IRTCNotificationContent extends RelationEvent { "m.mentions": IMentions; "decline_reason"?: string; "notification_type": RTCNotificationType; + /** + * The initial intent of the calling user. + */ + "m.call.intent"?: RTCCallIntent; "sender_ts": number; "lifetime": number; }