Implement decryption via the rust sdk (#3074)

A bunch of changes to tests, and wire up decryption.

spec/TestClient.ts

@@ -115,13 +115,12 @@ export class TestClient {
     }
 
     /**
-     * Set up expectations that the client will upload device keys.
+     * Set up expectations that the client will upload device keys (and possibly one-time keys)
      */
     public expectDeviceKeyUpload() {
         this.httpBackend
             .when("POST", "/keys/upload")
             .respond<IKeysUploadResponse, IUploadKeysRequest>(200, (_path, content) => {
-                expect(content.one_time_keys).toBe(undefined);
                 expect(content.device_keys).toBeTruthy();
 
                 logger.log(this + ": received device keys");
@@ -129,7 +128,17 @@ export class TestClient {
                 expect(Object.keys(this.oneTimeKeys!).length).toEqual(0);
 
                 this.deviceKeys = content.device_keys;
-                return { one_time_key_counts: { signed_curve25519: 0 } };
+
+                // the first batch of one-time keys may be uploaded at the same time.
+                if (content.one_time_keys) {
+                    logger.log(`${this}: received ${Object.keys(content.one_time_keys).length} one-time keys`);
+                    this.oneTimeKeys = content.one_time_keys;
+                }
+                return {
+                    one_time_key_counts: {
+                        signed_curve25519: Object.keys(this.oneTimeKeys!).length,
+                    },
+                };
             });
     }
 

spec/integ/megolm-integ.spec.ts

@@ -17,6 +17,8 @@ limitations under the License.
 
 import anotherjson from "another-json";
 import MockHttpBackend from "matrix-mock-request";
+import "fake-indexeddb/auto";
+import { IDBFactory } from "fake-indexeddb";
 
 import * as testUtils from "../test-utils/test-utils";
 import { TestClient } from "../TestClient";
@@ -38,9 +40,17 @@ import {
 } from "../../src/matrix";
 import { IDeviceKeys } from "../../src/crypto/dehydration";
 import { DeviceInfo } from "../../src/crypto/deviceinfo";
+import { CRYPTO_BACKENDS, InitCrypto } from "../test-utils/test-utils";
 
 const ROOM_ID = "!room:id";
 
+afterEach(() => {
+    // reset fake-indexeddb after each test, to make sure we don't leak connections
+    // cf https://github.com/dumbmatter/fakeIndexedDB#wipingresetting-the-indexeddb-for-a-fresh-state
+    // eslint-disable-next-line no-global-assign
+    indexedDB = new IDBFactory();
+});
+
 // start an Olm session with a given recipient
 async function createOlmSession(olmAccount: Olm.Account, recipientTestClient: TestClient): Promise<Olm.Session> {
     const keys = await recipientTestClient.awaitOneTimeKeyUpload();
@@ -341,11 +351,17 @@ async function expectSendMegolmMessage(
     return JSON.parse(r.plaintext);
 }
 
-describe("megolm", () => {
+describe.each(Object.entries(CRYPTO_BACKENDS))("megolm (%s)", (backend: string, initCrypto: InitCrypto) => {
     if (!global.Olm) {
+        // currently we use libolm to implement the crypto in the tests, so need it to be present.
         logger.warn("not running megolm tests: Olm not present");
         return;
     }
+
+    // oldBackendOnly is an alternative to `it` or `test` which will skip the test if we are running against the
+    // Rust backend. Once we have full support in the rust sdk, it will go away.
+    const oldBackendOnly = backend === "rust-sdk" ? test.skip : test;
+
     const Olm = global.Olm;
 
     let testOlmAccount = {} as unknown as Olm.Account;
@@ -410,8 +426,10 @@ describe("megolm", () => {
 
     beforeEach(async () => {
         aliceTestClient = new TestClient("@alice:localhost", "xzcvb", "akjgkrgjs");
-        await aliceTestClient.client.initCrypto();
+        await initCrypto(aliceTestClient.client);
 
+        // create a test olm device which we will use to communicate with alice. We use libolm to implement this.
+        await Olm.init();
         testOlmAccount = new Olm.Account();
         testOlmAccount.create();
         const testE2eKeys = JSON.parse(testOlmAccount.identity_keys());
@@ -424,13 +442,18 @@ describe("megolm", () => {
 
     it("Alice receives a megolm message", async () => {
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
+
         const p2pSession = await createOlmSession(testOlmAccount, aliceTestClient);
         const groupSession = new Olm.OutboundGroupSession();
         groupSession.create();
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
-
         // make the room_key event
         const roomKeyEncrypted = encryptGroupSessionKey({
             recipient: aliceTestClient,
@@ -472,16 +495,21 @@ describe("megolm", () => {
         expect(decryptedEvent.getContent().body).toEqual("42");
     });
 
-    it("Alice receives a megolm message before the session keys", async () => {
+    oldBackendOnly("Alice receives a megolm message before the session keys", async () => {
         // https://github.com/vector-im/element-web/issues/2273
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
+
         const p2pSession = await createOlmSession(testOlmAccount, aliceTestClient);
         const groupSession = new Olm.OutboundGroupSession();
         groupSession.create();
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
-
         // make the room_key event, but don't send it yet
         const roomKeyEncrypted = encryptGroupSessionKey({
             recipient: aliceTestClient,
@@ -535,13 +563,18 @@ describe("megolm", () => {
 
     it("Alice gets a second room_key message", async () => {
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
+
         const p2pSession = await createOlmSession(testOlmAccount, aliceTestClient);
         const groupSession = new Olm.OutboundGroupSession();
         groupSession.create();
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
-
         // make the room_key event
         const roomKeyEncrypted1 = encryptGroupSessionKey({
             recipient: aliceTestClient,
@@ -600,7 +633,7 @@ describe("megolm", () => {
         expect(event.getContent().body).toEqual("42");
     });
 
-    it("Alice sends a megolm message", async () => {
+    oldBackendOnly("Alice sends a megolm message", async () => {
         aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await aliceTestClient.start();
         const p2pSession = await establishOlmSession(aliceTestClient, testOlmAccount);
@@ -643,7 +676,7 @@ describe("megolm", () => {
         ]);
     });
 
-    it("We shouldn't attempt to send to blocked devices", async () => {
+    oldBackendOnly("We shouldn't attempt to send to blocked devices", async () => {
         aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await aliceTestClient.start();
         await establishOlmSession(aliceTestClient, testOlmAccount);
@@ -687,7 +720,7 @@ describe("megolm", () => {
             expect(() => aliceTestClient.client.getGlobalErrorOnUnknownDevices()).toThrowError("encryption disabled");
         });
 
-        it("should permit sending to unknown devices", async () => {
+        oldBackendOnly("should permit sending to unknown devices", async () => {
             expect(aliceTestClient.client.getGlobalErrorOnUnknownDevices()).toBeTruthy();
 
             aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
@@ -745,7 +778,7 @@ describe("megolm", () => {
             );
         });
 
-        it("should disable sending to unverified devices", async () => {
+        oldBackendOnly("should disable sending to unverified devices", async () => {
             aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
             await aliceTestClient.start();
             const p2pSession = await establishOlmSession(aliceTestClient, testOlmAccount);
@@ -803,7 +836,7 @@ describe("megolm", () => {
         });
     });
 
-    it("We should start a new megolm session when a device is blocked", async () => {
+    oldBackendOnly("We should start a new megolm session when a device is blocked", async () => {
         aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await aliceTestClient.start();
         const p2pSession = await establishOlmSession(aliceTestClient, testOlmAccount);
@@ -861,7 +894,7 @@ describe("megolm", () => {
     });
 
     // https://github.com/vector-im/element-web/issues/2676
-    it("Alice should send to her other devices", async () => {
+    oldBackendOnly("Alice should send to her other devices", async () => {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
         await aliceTestClient.start();
@@ -942,7 +975,7 @@ describe("megolm", () => {
         expect(decrypted.content?.body).toEqual("test");
     });
 
-    it("Alice should wait for device list to complete when sending a megolm message", async () => {
+    oldBackendOnly("Alice should wait for device list to complete when sending a megolm message", async () => {
         aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await aliceTestClient.start();
         await establishOlmSession(aliceTestClient, testOlmAccount);
@@ -972,15 +1005,20 @@ describe("megolm", () => {
         await Promise.all([downloadPromise, sendPromise]);
     });
 
-    it("Alice exports megolm keys and imports them to a new device", async () => {
+    oldBackendOnly("Alice exports megolm keys and imports them to a new device", async () => {
         aliceTestClient.expectKeyQuery({ device_keys: { "@alice:localhost": {} }, failures: {} });
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
+
         // establish an olm session with alice
         const p2pSession = await createOlmSession(testOlmAccount, aliceTestClient);
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
-
         const groupSession = new Olm.OutboundGroupSession();
         groupSession.create();
 
@@ -1027,11 +1065,15 @@ describe("megolm", () => {
         aliceTestClient.stop();
 
         aliceTestClient = new TestClient("@alice:localhost", "device2", "access_token2");
-        await aliceTestClient.client.initCrypto();
+        await initCrypto(aliceTestClient.client);
         await aliceTestClient.client.importRoomKeys(exported);
         await aliceTestClient.start();
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
 
         const syncResponse = {
             next_batch: 1,
@@ -1107,13 +1149,18 @@ describe("megolm", () => {
 
     it("Alice can decrypt a message with falsey content", async () => {
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto.deviceList.getUserByIdentityKey = () => "@bob:xyz";
+        }
+
         const p2pSession = await createOlmSession(testOlmAccount, aliceTestClient);
         const groupSession = new Olm.OutboundGroupSession();
         groupSession.create();
 
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => "@bob:xyz";
-
         // make the room_key event
         const roomKeyEncrypted = encryptGroupSessionKey({
             recipient: aliceTestClient,
@@ -1160,14 +1207,21 @@ describe("megolm", () => {
         expect(decryptedEvent.getClearContent()).toBeUndefined();
     });
 
-    it("Alice receives shared history before being invited to a room by the sharer", async () => {
+    oldBackendOnly("Alice receives shared history before being invited to a room by the sharer", async () => {
         const beccaTestClient = new TestClient("@becca:localhost", "foobar", "bazquux");
         await beccaTestClient.client.initCrypto();
 
         await aliceTestClient.start();
-        aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
         await beccaTestClient.start();
 
+        // if we're using the old crypto impl, stub out some methods in the device manager.
+        // TODO: replace this with intercepts of the /keys/query endpoint to make it impl agnostic.
+        if (aliceTestClient.client.crypto) {
+            aliceTestClient.client.crypto!.deviceList.downloadKeys = () => Promise.resolve({});
+            aliceTestClient.client.crypto!.deviceList.getDeviceByIdentityKey = () => device;
+            aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => beccaTestClient.client.getUserId()!;
+        }
+
         const beccaRoom = new Room(ROOM_ID, beccaTestClient.client, "@becca:localhost", {});
         beccaTestClient.client.store.storeRoom(beccaRoom);
         await beccaTestClient.client.setRoomEncryption(ROOM_ID, { algorithm: "m.megolm.v1.aes-sha2" });
@@ -1193,8 +1247,6 @@ describe("megolm", () => {
         event.claimedEd25519Key = null;
 
         const device = new DeviceInfo(beccaTestClient.client.deviceId!);
-        aliceTestClient.client.crypto!.deviceList.getDeviceByIdentityKey = () => device;
-        aliceTestClient.client.crypto!.deviceList.getUserByIdentityKey = () => beccaTestClient.client.getUserId()!;
 
         // Create an olm session for Becca and Alice's devices
         const aliceOtks = await aliceTestClient.awaitOneTimeKeyUpload();
@@ -1307,7 +1359,7 @@ describe("megolm", () => {
         await beccaTestClient.stop();
     });
 
-    it("Alice receives shared history before being invited to a room by someone else", async () => {
+    oldBackendOnly("Alice receives shared history before being invited to a room by someone else", async () => {
         const beccaTestClient = new TestClient("@becca:localhost", "foobar", "bazquux");
         await beccaTestClient.client.initCrypto();
 
@@ -1453,7 +1505,7 @@ describe("megolm", () => {
         await beccaTestClient.stop();
     });
 
-    it("allows sending an encrypted event as soon as room state arrives", async () => {
+    oldBackendOnly("allows sending an encrypted event as soon as room state arrives", async () => {
         /* Empirically, clients expect to be able to send encrypted events as soon as the
          * RoomStateEvent.NewMember notification is emitted, so test that works correctly.
          */
@@ -1578,7 +1630,7 @@ describe("megolm", () => {
             await aliceTestClient.httpBackend.flush(membersPath, 1);
         }
 
-        it("Sending an event initiates a member list sync", async () => {
+        oldBackendOnly("Sending an event initiates a member list sync", async () => {
             // we expect a call to the /members list...
             const memberListPromise = expectMembershipRequest(ROOM_ID, ["@bob:xyz"]);
 
@@ -1610,7 +1662,7 @@ describe("megolm", () => {
             ]);
         });
 
-        it("loading the membership list inhibits a later load", async () => {
+        oldBackendOnly("loading the membership list inhibits a later load", async () => {
             const room = aliceTestClient.client.getRoom(ROOM_ID)!;
             await Promise.all([room.loadMembersIfNeeded(), expectMembershipRequest(ROOM_ID, ["@bob:xyz"])]);
 
@@ -1642,7 +1694,7 @@ describe("megolm", () => {
         // TODO: there are a bunch more tests for this sort of thing in spec/unit/crypto/algorithms/megolm.spec.ts.
         //   They should be converted to integ tests and moved.
 
-        it("does not block decryption on an 'm.unavailable' report", async function () {
+        oldBackendOnly("does not block decryption on an 'm.unavailable' report", async function () {
             await aliceTestClient.start();
 
             // there may be a key downloads for alice

spec/test-utils/test-utils.ts

@@ -403,3 +403,15 @@ export const mkPusher = (extra: Partial<IPusher> = {}): IPusher => ({
     pushkey: "pushpush",
     ...extra,
 });
+
+/**
+ * a list of the supported crypto implementations, each with a callback to initialise that implementation
+ * for the given client
+ */
+export const CRYPTO_BACKENDS: Record<string, InitCrypto> = {};
+export type InitCrypto = (_: MatrixClient) => Promise<void>;
+
+CRYPTO_BACKENDS["rust-sdk"] = (client: MatrixClient) => client.initRustCrypto();
+if (global.Olm) {
+    CRYPTO_BACKENDS["libolm"] = (client: MatrixClient) => client.initCrypto();
+}

src/rust-crypto/rust-crypto.ts

@@ -16,6 +16,7 @@ limitations under the License.
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-js";
 import {
+    DecryptedRoomEvent,
     KeysBackupRequest,
     KeysClaimRequest,
     KeysQueryRequest,
@@ -76,8 +77,23 @@ export class RustCrypto implements CryptoBackend {
     }
 
     public async decryptEvent(event: MatrixEvent): Promise<IEventDecryptionResult> {
-        await this.olmMachine.decryptRoomEvent("event", new RustSdkCryptoJs.RoomId("room"));
+        const res = (await this.olmMachine.decryptRoomEvent(
-        throw new Error("not implemented");
+            JSON.stringify({
+                event_id: event.getId(),
+                type: event.getWireType(),
+                sender: event.getSender(),
+                state_key: event.getStateKey(),
+                content: event.getWireContent(),
+                origin_server_ts: event.getTs(),
+            }),
+            new RustSdkCryptoJs.RoomId(event.getRoomId()!),
+        )) as DecryptedRoomEvent;
+        return {
+            clearEvent: JSON.parse(res.event),
+            claimedEd25519Key: res.senderClaimedEd25519Key,
+            senderCurve25519Key: res.senderCurve25519Key,
+            forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+        };
     }
 
     public getEventEncryptionInfo(event: MatrixEvent): IEncryptedEventInfo {