Add call intent to RTC call notifications (#5010)

* Add media hint specifier

* Refactor to use m.call.intent and to apply to membership

* lint

* Add a mechanism to get the consensus of a call.

* Update tests

* Expose option to update the call intent.

* Better docs

* Add tests

* lint

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -222,6 +222,27 @@ describe("MatrixRTCSession", () => {
         });
     });
 
+    describe("getConsensusCallIntent", () => {
+        it.each([
+            [undefined, undefined, undefined],
+            ["audio", undefined, "audio"],
+            [undefined, "audio", "audio"],
+            ["audio", "audio", "audio"],
+            ["audio", "video", undefined],
+        ])("gets correct consensus for %s + %s = %s", (intentA, intentB, result) => {
+            jest.useFakeTimers();
+            jest.setSystemTime(4000);
+            const mockRoom = makeMockRoom([
+                Object.assign({}, membershipTemplate, { "m.call.intent": intentA }),
+                Object.assign({}, membershipTemplate, { "m.call.intent": intentB }),
+            ]);
+
+            sess = MatrixRTCSession.sessionForRoom(client, mockRoom, callSession);
+            expect(sess.getConsensusCallIntent()).toEqual(result);
+            jest.useRealTimers();
+        });
+    });
+
     describe("getsActiveFocus", () => {
         const firstPreferredFocus = {
             type: "livekit",
@@ -370,6 +391,79 @@ describe("MatrixRTCSession", () => {
             );
         });
 
+        it("sends a notification with a intent when starting a call and emits DidSendCallNotification", async () => {
+            // Simulate a join, including the update to the room state
+            // Ensure sendEvent returns event IDs so the DidSendCallNotification payload includes them
+            sendEventMock
+                .mockResolvedValueOnce({ event_id: "legacy-evt" })
+                .mockResolvedValueOnce({ event_id: "new-evt" });
+            const didSendEventFn = jest.fn();
+            sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, didSendEventFn);
+            // Create an additional listener to create a promise that resolves after the emission.
+            const didSendNotification = new Promise((resolve) => {
+                sess!.once(MatrixRTCSessionEvent.DidSendCallNotification, resolve);
+            });
+
+            sess!.joinRoomSession([mockFocus], mockFocus, { notificationType: "ring", callIntent: "audio" });
+            await Promise.race([sentStateEvent, new Promise((resolve) => setTimeout(resolve, 5000))]);
+
+            mockRoomState(mockRoom, [
+                {
+                    ...membershipTemplate,
+                    "user_id": client.getUserId()!,
+                    // This is what triggers the intent type on the notification event.
+                    "m.call.intent": "audio",
+                },
+            ]);
+
+            sess!.onRTCSessionMemberUpdate();
+            const ownMembershipId = sess?.memberships[0].eventId;
+            expect(sess!.getConsensusCallIntent()).toEqual("audio");
+
+            expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.RTCNotification, {
+                "m.mentions": { user_ids: [], room: true },
+                "notification_type": "ring",
+                "m.call.intent": "audio",
+                "m.relates_to": {
+                    event_id: ownMembershipId,
+                    rel_type: "m.reference",
+                },
+                "lifetime": 30000,
+                "sender_ts": expect.any(Number),
+            });
+
+            // Check if deprecated notify event is also sent.
+            expect(client.sendEvent).toHaveBeenCalledWith(mockRoom!.roomId, EventType.CallNotify, {
+                "application": "m.call",
+                "m.mentions": { user_ids: [], room: true },
+                "notify_type": "ring",
+                "call_id": "",
+            });
+            await didSendNotification;
+            // And ensure we emitted the DidSendCallNotification event with both payloads
+            expect(didSendEventFn).toHaveBeenCalledWith(
+                {
+                    "event_id": "new-evt",
+                    "lifetime": 30000,
+                    "m.mentions": { room: true, user_ids: [] },
+                    "m.relates_to": {
+                        event_id: expect.any(String),
+                        rel_type: "m.reference",
+                    },
+                    "notification_type": "ring",
+                    "m.call.intent": "audio",
+                    "sender_ts": expect.any(Number),
+                },
+                {
+                    "application": "m.call",
+                    "call_id": "",
+                    "event_id": "legacy-evt",
+                    "m.mentions": { room: true, user_ids: [] },
+                    "notify_type": "ring",
+                },
+            );
+        });
+
         it("doesn't send a notification when joining an existing call", async () => {
             // Add another member to the call so that it is considered an existing call
             mockRoomState(mockRoom, [membershipTemplate]);

spec/unit/matrixrtc/MembershipManager.spec.ts

@@ -901,6 +901,43 @@ describe("MembershipManager", () => {
             }
         });
     });
+
+    describe("updateCallIntent()", () => {
+        it("should fail if the user has not joined the call", async () => {
+            const manager = new MembershipManager({}, room, client, () => undefined, callSession);
+            // After joining we want our own focus to be the one we select.
+            try {
+                await manager.updateCallIntent("video");
+                throw Error("Should have thrown");
+            } catch {}
+        });
+
+        it("can adjust the intent", async () => {
+            const manager = new MembershipManager({}, room, client, () => undefined, callSession);
+            manager.join([]);
+            expect(manager.isActivated()).toEqual(true);
+            const membership = mockCallMembership({ ...membershipTemplate, user_id: client.getUserId()! }, room.roomId);
+            await manager.onRTCSessionMemberUpdate([membership]);
+            await manager.updateCallIntent("video");
+            expect(client.sendStateEvent).toHaveBeenCalledTimes(2);
+            const eventContent = (client.sendStateEvent as Mock).mock.calls[0][2] as SessionMembershipData;
+            expect(eventContent["created_ts"]).toEqual(membership.createdTs());
+            expect(eventContent["m.call.intent"]).toEqual("video");
+        });
+
+        it("does nothing if the intent doesn't change", async () => {
+            const manager = new MembershipManager({ callIntent: "video" }, room, client, () => undefined, callSession);
+            manager.join([]);
+            expect(manager.isActivated()).toEqual(true);
+            const membership = mockCallMembership(
+                { ...membershipTemplate, "user_id": client.getUserId()!, "m.call.intent": "video" },
+                room.roomId,
+            );
+            await manager.onRTCSessionMemberUpdate([membership]);
+            await manager.updateCallIntent("video");
+            expect(client.sendStateEvent).toHaveBeenCalledTimes(0);
+        });
+    });
 });
 
 it("Should prefix log with MembershipManager used", () => {

src/matrixrtc/CallMembership.ts

@@ -19,6 +19,7 @@ import { deepCompare } from "../utils.ts";
 import { type Focus } from "./focus.ts";
 import { isLivekitFocusActive } from "./LivekitFocus.ts";
 import { type SessionDescription } from "./MatrixRTCSession.ts";
+import { type RTCCallIntent } from "./types.ts";
 
 /**
  * The default duration in milliseconds that a membership is considered valid for.
@@ -31,13 +32,13 @@ type CallScope = "m.room" | "m.user";
 
 /**
  * MSC4143 (MatrixRTC) session membership data.
- * Represents an entry in the memberships section of an m.call.member event as it is on the wire.
+ * Represents the `session` in the memberships section of an m.call.member event as it is on the wire.
  **/
 export type SessionMembershipData = {
     /**
      * The RTC application defines the type of the RTC session.
      */
-    application: string;
+    "application": string;
 
     /**
      * The id of this session.
@@ -45,23 +46,23 @@ export type SessionMembershipData = {
      * multiple session in one room. A room wide session that is not associated with a user,
      * and therefore immune to creation race conflicts, uses the `call_id: ""`.
      */
-    call_id: string;
+    "call_id": string;
 
     /**
      * The Matrix device ID of this session. A single user can have multiple sessions on different devices.
      */
-    device_id: string;
+    "device_id": string;
 
     /**
      * The focus selection system this user/membership is using.
      */
-    focus_active: Focus;
+    "focus_active": Focus;
 
     /**
      * A list of possible foci this uses knows about. One of them might be used based on the focus_active
      * selection system.
      */
-    foci_preferred: Focus[];
+    "foci_preferred": Focus[];
 
     /**
      * Optional field that contains the creation of the session. If it is undefined the creation
@@ -70,7 +71,7 @@ export type SessionMembershipData = {
      *  - If it is undefined it can be interpreted as a "Join".
      *  - If it is defined it can be interpreted as an "Update"
      */
-    created_ts?: number;
+    "created_ts"?: number;
 
     // Application specific data
 
@@ -78,17 +79,26 @@ export type SessionMembershipData = {
      * If the `application` = `"m.call"` this defines if it is a room or user owned call.
      * There can always be one room scroped call but multiple user owned calls (breakout sessions)
      */
-    scope?: CallScope;
+    "scope"?: CallScope;
 
     /**
      * Optionally we allow to define a delta to the `created_ts` that defines when the event is expired/invalid.
      * This should be set to multiple hours. The only reason it exist is to deal with failed delayed events.
      * (for example caused by a homeserver crashes)
      **/
-    expires?: number;
+    "expires"?: number;
+
+    /**
+     * The intent of the call from the perspective of this user. This may be an audio call, video call or
+     * something else.
+     */
+    "m.call.intent"?: RTCCallIntent;
 };
 
-const checkSessionsMembershipData = (data: any, errors: string[]): data is SessionMembershipData => {
+const checkSessionsMembershipData = (
+    data: Partial<Record<keyof SessionMembershipData, any>>,
+    errors: string[],
+): data is SessionMembershipData => {
     const prefix = "Malformed session membership event: ";
     if (typeof data.device_id !== "string") errors.push(prefix + "device_id must be string");
     if (typeof data.call_id !== "string") errors.push(prefix + "call_id must be string");
@@ -96,10 +106,17 @@ const checkSessionsMembershipData = (data: any, errors: string[]): data is Sessi
     if (typeof data.focus_active?.type !== "string") errors.push(prefix + "focus_active.type must be a string");
     if (!Array.isArray(data.foci_preferred)) errors.push(prefix + "foci_preferred must be an array");
     // optional parameters
-    if (data.created_ts && typeof data.created_ts !== "number") errors.push(prefix + "created_ts must be number");
+    if (data.created_ts !== undefined && typeof data.created_ts !== "number") {
+        errors.push(prefix + "created_ts must be number");
+    }
 
     // application specific data (we first need to check if they exist)
-    if (data.scope && typeof data.scope !== "string") errors.push(prefix + "scope must be string");
+    if (data.scope !== undefined && typeof data.scope !== "string") errors.push(prefix + "scope must be string");
+
+    if (data["m.call.intent"] !== undefined && typeof data["m.call.intent"] !== "string") {
+        errors.push(prefix + "m.call.intent must be a string");
+    }
+
     return errors.length === 0;
 };
 
@@ -142,6 +159,10 @@ export class CallMembership {
         return this.membershipData.device_id;
     }
 
+    public get callIntent(): RTCCallIntent | undefined {
+        return this.membershipData["m.call.intent"];
+    }
+
     public get sessionDescription(): SessionDescription {
         return {
             application: this.membershipData.application,

src/matrixrtc/IMembershipManager.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import type { CallMembership } from "./CallMembership.ts";
 import type { Focus } from "./focus.ts";
-import type { Status } from "./types.ts";
+import type { RTCCallIntent, Status } from "./types.ts";
 import { type TypedEventEmitter } from "../models/typed-event-emitter.ts";
 
 export enum MembershipManagerEvent {
@@ -100,4 +100,10 @@ export interface IMembershipManager
      * @returns the used active focus in the currently joined session or undefined if not joined.
      */
     getActiveFocus(): Focus | undefined;
+
+    /**
+     * Update the intent of a membership on the call (e.g. user is now providing a video feed)
+     * @param callIntent The new intent to set.
+     */
+    updateCallIntent(callIntent: RTCCallIntent): Promise<void>;
 }

src/matrixrtc/MatrixRTCSession.ts

@@ -34,6 +34,7 @@ import {
     type Status,
     type IRTCNotificationContent,
     type ICallNotifyContent,
+    type RTCCallIntent,
 } from "./types.ts";
 import { RoomKeyTransport } from "./RoomKeyTransport.ts";
 import {
@@ -92,6 +93,11 @@ export interface SessionConfig {
      * @default `undefined` (no notification)
      */
     notificationType?: RTCNotificationType;
+
+    /**
+     * Determines the kind of call this will be.
+     */
+    callIntent?: RTCCallIntent;
 }
 
 /**
@@ -614,6 +620,32 @@ export class MatrixRTCSession extends TypedEventEmitter<
         return this.memberships[0];
     }
 
+    /**
+     * Get the call intent for the current call, based on what members are advertising. If one or more
+     * members disagree on the current call intent, or nobody specifies one then `undefined` is returned.
+     *
+     * If all members that specify a call intent agree, that value is returned.
+     * @returns A call intent, or `undefined` if no consensus or not given.
+     */
+    public getConsensusCallIntent(): RTCCallIntent | undefined {
+        const getFirstCallIntent = this.memberships.find((m) => !!m.callIntent)?.callIntent;
+        if (!getFirstCallIntent) {
+            return undefined;
+        }
+        if (this.memberships.every((m) => !m.callIntent || m.callIntent === getFirstCallIntent)) {
+            return getFirstCallIntent;
+        }
+        return undefined;
+    }
+
+    public async updateCallIntent(callIntent: RTCCallIntent): Promise<void> {
+        const myMembership = this.membershipManager?.ownMembership;
+        if (!myMembership) {
+            throw Error("Not connected yet");
+        }
+        await this.membershipManager?.updateCallIntent(callIntent);
+    }
+
     /**
      * This method is used when the user is not yet connected to the Session but wants to know what focus
      * the users in the session are using to make a decision how it wants/should connect.
@@ -665,9 +697,17 @@ export class MatrixRTCSession extends TypedEventEmitter<
     }
 
     /**
-     * Sends a notification corresponding to the configured notify type.
+     * Sends notification events to indiciate the call has started.
+     * Note: This does not return a promise, instead scheduling the notification events to be sent.
+     * @param parentEventId Event id linking to your RTC call membership event.
+     * @param notificationType The type of notification to send
+     * @param callIntent The type of call this is (e.g. "audio").
      */
-    private sendCallNotify(parentEventId: string, notificationType: RTCNotificationType): void {
+    private sendCallNotify(
+        parentEventId: string,
+        notificationType: RTCNotificationType,
+        callIntent?: RTCCallIntent,
+    ): void {
         const sendLegacyNotificationEvent = async (): Promise<{
             response: ISendEventResponse;
             content: ICallNotifyContent;
@@ -695,6 +735,9 @@ export class MatrixRTCSession extends TypedEventEmitter<
                 "sender_ts": Date.now(),
                 "lifetime": 30_000, // 30 seconds
             };
+            if (callIntent) {
+                content["m.call.intent"] = callIntent;
+            }
             const response = await this.client.sendEvent(this.roomSubset.roomId, EventType.RTCNotification, content);
             return { response, content };
         };
@@ -757,7 +800,11 @@ export class MatrixRTCSession extends TypedEventEmitter<
                 // If we're the first member in the call, we're responsible for
                 // sending the notification event
                 if (ownMembership.eventId && this.joinConfig?.notificationType) {
-                    this.sendCallNotify(ownMembership.eventId, this.joinConfig.notificationType);
+                    this.sendCallNotify(
+                        ownMembership.eventId,
+                        this.joinConfig.notificationType,
+                        ownMembership.callIntent,
+                    );
                 } else {
                     this.logger.warn("Own membership eventId is undefined, cannot send call notification");
                 }

src/matrixrtc/MembershipManager.ts

@@ -24,9 +24,9 @@ import { type Logger, logger as rootLogger } from "../logger.ts";
 import { type Room } from "../models/room.ts";
 import { type CallMembership, DEFAULT_EXPIRE_DURATION, type SessionMembershipData } from "./CallMembership.ts";
 import { type Focus } from "./focus.ts";
-import { isMyMembership, Status } from "./types.ts";
+import { isMyMembership, type RTCCallIntent, Status } from "./types.ts";
 import { isLivekitFocusActive } from "./LivekitFocus.ts";
-import { type SessionDescription, type MembershipConfig } from "./MatrixRTCSession.ts";
+import { type SessionDescription, type MembershipConfig, type SessionConfig } from "./MatrixRTCSession.ts";
 import { ActionScheduler, type ActionUpdate } from "./MembershipManagerActionScheduler.ts";
 import { TypedEventEmitter } from "../models/typed-event-emitter.ts";
 import {
@@ -156,6 +156,7 @@ export class MembershipManager
 {
     private activated = false;
     private logger: Logger;
+    private callIntent: RTCCallIntent | undefined;
 
     public isActivated(): boolean {
         return this.activated;
@@ -230,7 +231,10 @@ export class MembershipManager
 
     private leavePromiseResolvers?: PromiseWithResolvers<boolean>;
 
-    public async onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise<void> {
+    public onRTCSessionMemberUpdate(memberships: CallMembership[]): Promise<void> {
+        if (!this.isActivated()) {
+            return Promise.resolve();
+        }
         const userId = this.client.getUserId();
         const deviceId = this.client.getDeviceId();
         if (!userId || !deviceId) {
@@ -239,7 +243,7 @@ export class MembershipManager
         }
         this._ownMembership = memberships.find((m) => isMyMembership(m, userId, deviceId));
 
-        if (this.isActivated() && !this._ownMembership) {
+        if (!this._ownMembership) {
             // If one of these actions are scheduled or are getting inserted in the next iteration, we should already
             // take care of our missing membership.
             const sendingMembershipActions = [
@@ -281,6 +285,18 @@ export class MembershipManager
         }
     }
 
+    public async updateCallIntent(callIntent: RTCCallIntent): Promise<void> {
+        if (!this.activated || !this.ownMembership) {
+            throw Error("You cannot update your intent before joining the call");
+        }
+        if (this.ownMembership.callIntent === callIntent) {
+            return; // No-op
+        }
+        this.callIntent = callIntent;
+        // Kick off a new membership event as a result.
+        await this.sendJoinEvent();
+    }
+
     /**
      * @throws if the client does not return user or device id.
      * @param joinConfig
@@ -289,7 +305,7 @@ export class MembershipManager
      * @param getOldestMembership
      */
     public constructor(
-        private joinConfig: MembershipConfig | undefined,
+        private joinConfig: (SessionConfig & MembershipConfig) | undefined,
         private room: Pick<Room, "getLiveTimeline" | "roomId" | "getVersion">,
         private client: Pick<
             MatrixClient,
@@ -311,6 +327,7 @@ export class MembershipManager
         this.deviceId = deviceId;
         this.stateKey = this.makeMembershipStateKey(userId, deviceId);
         this.state = MembershipManager.defaultState;
+        this.callIntent = joinConfig?.callIntent;
         this.scheduler = new ActionScheduler((type): Promise<ActionUpdate> => {
             if (this.oldStatus) {
                 // we put this at the beginning of the actions scheduler loop handle callback since it is a loop this
@@ -741,15 +758,18 @@ export class MembershipManager
      * Constructs our own membership
      */
     private makeMyMembership(expires: number): SessionMembershipData {
+        const hasPreviousEvent = !!this.ownMembership;
         return {
             // TODO: use the new format for m.rtc.member events where call_id becomes session.id
-            application: this.sessionDescription.application,
-            call_id: this.sessionDescription.id,
-            scope: "m.room",
-            device_id: this.deviceId,
+            "application": this.sessionDescription.application,
+            "call_id": this.sessionDescription.id,
+            "scope": "m.room",
+            "device_id": this.deviceId,
             expires,
-            focus_active: { type: "livekit", focus_selection: "oldest_membership" },
-            foci_preferred: this.fociPreferred ?? [],
+            "focus_active": { type: "livekit", focus_selection: "oldest_membership" },
+            "foci_preferred": this.fociPreferred ?? [],
+            "m.call.intent": this.callIntent,
+            ...(hasPreviousEvent ? { created_ts: this.ownMembership?.createdTs() } : undefined),
         };
     }
 

src/matrixrtc/types.ts

@@ -96,10 +96,20 @@ export interface ICallNotifyContent {
 }
 
 export type RTCNotificationType = "ring" | "notification";
+
+/**
+ * Represents the intention of the call from the perspective of the sending user.
+ * May be any string, although `"audio"` and `"video"` are commonly accepted values.
+ */
+export type RTCCallIntent = "audio" | "video" | string;
 export interface IRTCNotificationContent extends RelationEvent {
     "m.mentions": IMentions;
     "decline_reason"?: string;
     "notification_type": RTCNotificationType;
+    /**
+     * The initial intent of the calling user.
+     */
+    "m.call.intent"?: RTCCallIntent;
     "sender_ts": number;
     "lifetime": number;
 }