Support for migration from from libolm (#3978)

* Use a `StoreHandle` to init OlmMachine

This will be faster if we need to prepare the store.

* Include "needsBackup" flag in inbound group session batches

* On startup, import data from libolm cryptostore

* ISessionExtended -> SessionExtended

.prettierignore

@@ -25,5 +25,6 @@ out
 # This file is owned, parsed, and generated by allchange, which doesn't comply with prettier
 /CHANGELOG.md
 
-# This file is also autogenerated
+# These files are also autogenerated
 /spec/test-utils/test-data/index.ts
+/spec/test-utils/test_indexeddb_cryptostore_dump/dump.json

spec/integ/crypto/rust-crypto.spec.ts

@@ -16,8 +16,12 @@ limitations under the License.
 
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
+import fetchMock from "fetch-mock-jest";
 
-import { createClient } from "../../../src";
+import { createClient, IndexedDBCryptoStore } from "../../../src";
+import { populateStore } from "../../test-utils/test_indexeddb_cryptostore_dump";
+
+jest.setTimeout(15000);
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -88,6 +92,47 @@ describe("MatrixClient.initRustCrypto", () => {
         await matrixClient.initRustCrypto();
         await matrixClient.initRustCrypto();
     });
+
+    it("should migrate from libolm", async () => {
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+            auth_data: {
+                public_key: "q+HZiJdHl2Yopv9GGvv7EYSzDMrAiRknK4glSdoaomI",
+                signatures: {
+                    "@vdhtest200713:matrix.org": {
+                        "ed25519:gh9fGr39eNZUdWynEMJ/q/WZq/Pk/foFxHXFBFm18ZI":
+                            "reDp6Mu+j+tfUL3/T6f5OBT3N825Lzpc43vvG+RvjX6V+KxXzodBQArgCoeEHLtL9OgSBmNrhTkSOX87MWCKAw",
+                        "ed25519:KMFSTJSMLB":
+                            "F8tyV5W6wNi0GXTdSg+gxSCULQi0EYxdAAqfkyNq58KzssZMw5i+PRA0aI2b+D7NH/aZaJrtiYNHJ0gWLSQvAw",
+                    },
+                },
+            },
+            version: "7",
+            algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+            etag: "1",
+            count: 79,
+        });
+
+        const testStoreName = "test-store";
+        await populateStore(testStoreName);
+        const cryptoStore = new IndexedDBCryptoStore(indexedDB, testStoreName);
+
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@vdhtest200713:matrix.org",
+            deviceId: "KMFSTJSMLB",
+            cryptoStore,
+            pickleKey: "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o",
+        });
+
+        await matrixClient.initRustCrypto();
+
+        // Do some basic checks on the imported data
+        const deviceKeys = await matrixClient.getCrypto()!.getOwnDeviceKeys();
+        expect(deviceKeys.curve25519).toEqual("LKv0bKbc0EC4h0jknbemv3QalEkeYvuNeUXVRgVVTTU");
+        expect(deviceKeys.ed25519).toEqual("qK70DEqIXq7T+UU3v/al47Ab4JkMEBLpNrTBMbS5rrw");
+
+        expect(await matrixClient.getCrypto()!.getActiveSessionBackupVersion()).toEqual("7");
+    });
 });
 
 describe("MatrixClient.clearStores", () => {

spec/test-utils/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,53 @@
+## Dump of libolm indexeddb cryptostore
+
+This directory contains a dump of a real indexeddb store from a session using
+libolm crypto.
+
+The corresponding pickle key is `+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o`.
+
+It was created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

spec/test-utils/test_indexeddb_cryptostore_dump/index.ts

@@ -0,0 +1,136 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+/**
+ * Populate an IndexedDB store with the test data from this directory.
+ *
+ * @param name - Name of the IndexedDB database to create.
+ */
+export async function populateStore(name: string): Promise<IDBDatabase> {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise<IDBDatabase>((resolve, reject) => {
+        req.onupgradeneeded = (ev): void => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev): void => {
+            reject(req.error);
+        };
+
+        req.onsuccess = (): void => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(db);
+
+    return db;
+}
+
+/** Create the schema for the indexed db store */
+function upgradeDatabase(oldVersion: number, db: IDBDatabase) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+async function importData(db: IDBDatabase) {
+    const path = resolve("spec/test-utils/test_indexeddb_cryptostore_dump/dump.json");
+    const json: Record<string, Array<{ key?: any; value: any }>> = JSON.parse(
+        await readFile(path, { encoding: "utf8" }),
+    );
+
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx: number) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}

spec/unit/crypto/store/CryptoStore.spec.ts

@@ -145,6 +145,11 @@ describe.each([
             const N_SESSIONS_PER_DEVICE = 6;
             await createSessions(N_DEVICES, N_SESSIONS_PER_DEVICE);
 
+            // Mark one of the sessions as needing backup
+            await store.doTxn("readwrite", IndexedDBCryptoStore.STORE_BACKUP, async (txn) => {
+                await store.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }], txn);
+            });
+
             const batch = await store.getEndToEndInboundGroupSessionsBatch();
             expect(batch!.length).toEqual(N_DEVICES * N_SESSIONS_PER_DEVICE);
             for (let i = 0; i < N_DEVICES; i++) {
@@ -153,6 +158,9 @@ describe.each([
 
                     expect(r.senderKey).toEqual(pad43(`device${i}`));
                     expect(r.sessionId).toEqual(`session${j}`);
+
+                    // only the last session needs backup
+                    expect(r.needsBackup).toBe(i === 5 && j === 5);
                 }
             }
         });

spec/unit/rust-crypto/rust-crypto.spec.ts

@@ -15,7 +15,15 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
-import { KeysQueryRequest, OlmMachine } from "@matrix-org/matrix-sdk-crypto-wasm";
+import {
+    BaseMigrationData,
+    KeysQueryRequest,
+    Migration,
+    OlmMachine,
+    PickledInboundGroupSession,
+    PickledSession,
+    StoreHandle,
+} from "@matrix-org/matrix-sdk-crypto-wasm";
 import { mocked, Mocked } from "jest-mock";
 import fetchMock from "fetch-mock-jest";
 
@@ -25,6 +33,7 @@ import {
     CryptoEvent,
     Device,
     DeviceVerification,
+    encodeBase64,
     HttpApiEvent,
     HttpApiEventHandlerMap,
     IHttpOpts,
@@ -32,6 +41,7 @@ import {
     MatrixClient,
     MatrixEvent,
     MatrixHttpApi,
+    MemoryCryptoStore,
     TypedEventEmitter,
 } from "../../../src";
 import { mkEvent } from "../../test-utils/test-utils";
@@ -59,6 +69,8 @@ import { logger } from "../../../src/logger";
 import { OutgoingRequestsManager } from "../../../src/rust-crypto/OutgoingRequestsManager";
 import { ClientEvent, ClientEventHandlerMap } from "../../../src/client";
 import { Curve25519AuthData } from "../../../src/crypto-api/keybackup";
+import { encryptAES } from "../../../src/crypto/aes";
+import { CryptoStore, SecretStorePrivateKeys } from "../../../src/crypto/store/base";
 
 const TEST_USER = "@alice:example.com";
 const TEST_DEVICE_ID = "TEST_DEVICE";
@@ -73,16 +85,22 @@ describe("initRustCrypto", () => {
         return {
             registerRoomKeyUpdatedCallback: jest.fn(),
             registerUserIdentityUpdatedCallback: jest.fn(),
-            getSecretsFromInbox: jest.fn().mockResolvedValue(["dGhpc2lzYWZha2VzZWNyZXQ="]),
+            getSecretsFromInbox: jest.fn().mockResolvedValue([]),
             deleteSecretsFromInbox: jest.fn(),
             registerReceiveSecretCallback: jest.fn(),
             outgoingRequests: jest.fn(),
+            isBackupEnabled: jest.fn().mockResolvedValue(false),
+            verifyBackup: jest.fn().mockResolvedValue({ trusted: jest.fn().mockReturnValue(false) }),
+            getBackupKeys: jest.fn(),
         } as unknown as Mocked<OlmMachine>;
     }
 
     it("passes through the store params", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "init_from_store").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -95,17 +113,16 @@ describe("initRustCrypto", () => {
             storePassphrase: "storePassphrase",
         });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            "storePrefix",
-            "storePassphrase",
-        );
+        expect(StoreHandle.open).toHaveBeenCalledWith("storePrefix", "storePassphrase");
+        expect(OlmMachine.init_from_store).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("suppresses the storePassphrase if storePrefix is unset", async () => {
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
         const testOlmMachine = makeTestOlmMachine();
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        jest.spyOn(OlmMachine, "init_from_store").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -118,12 +135,16 @@ describe("initRustCrypto", () => {
             storePassphrase: "storePassphrase",
         });
 
-        expect(OlmMachine.initialize).toHaveBeenCalledWith(expect.anything(), expect.anything(), undefined, undefined);
+        expect(StoreHandle.open).toHaveBeenCalledWith(undefined, undefined);
+        expect(OlmMachine.init_from_store).toHaveBeenCalledWith(expect.anything(), expect.anything(), mockStore);
     });
 
     it("Should get secrets from inbox on start", async () => {
-        const testOlmMachine = makeTestOlmMachine() as OlmMachine;
-        jest.spyOn(OlmMachine, "initialize").mockResolvedValue(testOlmMachine);
+        const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+        jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
+        const testOlmMachine = makeTestOlmMachine();
+        jest.spyOn(OlmMachine, "init_from_store").mockResolvedValue(testOlmMachine);
 
         await initRustCrypto({
             logger,
@@ -138,6 +159,156 @@ describe("initRustCrypto", () => {
 
         expect(testOlmMachine.getSecretsFromInbox).toHaveBeenCalledWith("m.megolm_backup.v1");
     });
+
+    describe("libolm migration", () => {
+        it("migrates data from a legacy crypto store", async () => {
+            const PICKLE_KEY = "pickle1234";
+            const legacyStore = new MemoryCryptoStore();
+
+            // Populate the legacy store with some test data
+            const storeSecretKey = (type: string, key: string) =>
+                encryptAndStoreSecretKey(type, new TextEncoder().encode(key), PICKLE_KEY, legacyStore);
+
+            await legacyStore.storeAccount({}, "not a real account");
+            await storeSecretKey("m.megolm_backup.v1", "backup key");
+            await storeSecretKey("master", "master key");
+            await storeSecretKey("self_signing", "ssk");
+            await storeSecretKey("user_signing", "usk");
+            const nDevices = 6;
+            const nSessionsPerDevice = 10;
+            createSessions(legacyStore, nDevices, nSessionsPerDevice);
+            createMegolmSessions(legacyStore, nDevices, nSessionsPerDevice);
+            await legacyStore.markSessionsNeedingBackup([{ senderKey: pad43("device5"), sessionId: "session5" }]);
+
+            // Stub out a bunch of stuff in the Rust library
+            const mockStore = { free: jest.fn() } as unknown as StoreHandle;
+            jest.spyOn(StoreHandle, "open").mockResolvedValue(mockStore);
+
+            jest.spyOn(Migration, "migrateBaseData").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateOlmSessions").mockResolvedValue(undefined);
+            jest.spyOn(Migration, "migrateMegolmSessions").mockResolvedValue(undefined);
+
+            const testOlmMachine = makeTestOlmMachine();
+            jest.spyOn(OlmMachine, "init_from_store").mockResolvedValue(testOlmMachine);
+
+            fetchMock.get("path:/_matrix/client/v3/room_keys/version", { version: "45" });
+
+            await initRustCrypto({
+                logger,
+                http: makeMatrixHttpApi(),
+                userId: TEST_USER,
+                deviceId: TEST_DEVICE_ID,
+                secretStorage: {} as ServerSideSecretStorage,
+                cryptoCallbacks: {} as CryptoCallbacks,
+                storePrefix: "storePrefix",
+                storePassphrase: "storePassphrase",
+                legacyCryptoStore: legacyStore,
+                legacyPickleKey: PICKLE_KEY,
+            });
+
+            // Check that the migration functions were correctly called
+            expect(Migration.migrateBaseData).toHaveBeenCalledWith(
+                expect.any(BaseMigrationData),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            const data = mocked(Migration.migrateBaseData).mock.calls[0][0];
+            expect(data.pickledAccount).toEqual("not a real account");
+            expect(data.userId!.toString()).toEqual(TEST_USER);
+            expect(data.deviceId!.toString()).toEqual(TEST_DEVICE_ID);
+            expect(atob(data.backupRecoveryKey!)).toEqual("backup key");
+            expect(data.backupVersion).toEqual("45");
+            expect(atob(data.privateCrossSigningMasterKey!)).toEqual("master key");
+            expect(atob(data.privateCrossSigningUserSigningKey!)).toEqual("usk");
+            expect(atob(data.privateCrossSigningSelfSigningKey!)).toEqual("ssk");
+
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateOlmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const sessions1: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[0][0];
+            expect(sessions1.length).toEqual(50);
+            const sessions2: PickledSession[] = mocked(Migration.migrateOlmSessions).mock.calls[1][0];
+            expect(sessions2.length).toEqual(10);
+            const sessions = [...sessions1, ...sessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = sessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(`device${i}`);
+                    expect(session.pickle).toEqual(`session${i}.${j}`);
+                    expect(session.creationTime).toEqual(new Date(1000));
+                    expect(session.lastUseTime).toEqual(new Date(1000));
+                }
+            }
+
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledTimes(2);
+            expect(Migration.migrateMegolmSessions).toHaveBeenCalledWith(
+                expect.any(Array),
+                new Uint8Array(Buffer.from(PICKLE_KEY)),
+                mockStore,
+            );
+            // First call should have 50 entries; second should have 10
+            const megolmSessions1: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[0][0];
+            expect(megolmSessions1.length).toEqual(50);
+            const megolmSessions2: PickledInboundGroupSession[] = mocked(Migration.migrateMegolmSessions).mock
+                .calls[1][0];
+            expect(megolmSessions2.length).toEqual(10);
+            const megolmSessions = [...megolmSessions1, ...megolmSessions2];
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const session = megolmSessions[i * nSessionsPerDevice + j];
+                    expect(session.senderKey).toEqual(pad43(`device${i}`));
+                    expect(session.pickle).toEqual("sessionPickle");
+                    expect(session.roomId!.toString()).toEqual("!room:id");
+                    // only one of the sessions needs backing up
+                    expect(session.backedUp).toEqual(i !== 5 || j !== 5);
+                }
+            }
+        }, 10000);
+
+        async function encryptAndStoreSecretKey(type: string, key: Uint8Array, pickleKey: string, store: CryptoStore) {
+            const encryptedKey = await encryptAES(encodeBase64(key), Buffer.from(pickleKey), type);
+            store.storeSecretStorePrivateKey(undefined, type as keyof SecretStorePrivateKeys, encryptedKey);
+        }
+
+        /** Create a bunch of fake Olm sessions and stash them in the DB. */
+        function createSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    const sessionData = {
+                        deviceKey: `device${i}`,
+                        sessionId: `session${j}`,
+                        session: `session${i}.${j}`,
+                        lastReceivedMessageTs: 1000,
+                    };
+                    store.storeEndToEndSession(`device${i}`, `session${j}`, sessionData, undefined);
+                }
+            }
+        }
+
+        /** Create a bunch of fake Megolm sessions and stash them in the DB. */
+        function createMegolmSessions(store: CryptoStore, nDevices: number, nSessionsPerDevice: number) {
+            for (let i = 0; i < nDevices; i++) {
+                for (let j = 0; j < nSessionsPerDevice; j++) {
+                    store.storeEndToEndInboundGroupSession(
+                        pad43(`device${i}`),
+                        `session${j}`,
+                        {
+                            forwardingCurve25519KeyChain: [],
+                            keysClaimed: { ed25519: "sender_signing_key" },
+                            room_id: "!room:id",
+                            session: "sessionPickle",
+                        },
+                        undefined,
+                    );
+                }
+            }
+        }
+    });
 });
 
 describe("RustCrypto", () => {
@@ -1095,3 +1266,8 @@ class DummyAccountDataClient
         return {};
     }
 }
+
+/** Pad a string to 43 characters long */
+function pad43(x: string): string {
+    return x + ".".repeat(43 - x.length);
+}

src/client.ts

@@ -2328,7 +2328,10 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
             cryptoCallbacks: this.cryptoCallbacks,
             storePrefix: useIndexedDB ? RUST_SDK_STORE_PREFIX : null,
             storePassphrase: this.pickleKey,
+            legacyCryptoStore: this.cryptoStore,
+            legacyPickleKey: this.pickleKey ?? "DEFAULT_KEY",
         });
+
         rustCrypto.setSupportedVerificationMethods(this.verificationMethods);
 
         this.cryptoBackend = rustCrypto;

src/crypto/store/base.ts

@@ -182,7 +182,7 @@ export interface CryptoStore {
      * @returns A batch of Megolm Sessions, or `null` if no sessions are left.
      * @internal
      */
-    getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null>;
+    getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null>;
 
     /**
      * Delete a batch of Megolm sessions from the database.
@@ -223,6 +223,11 @@ export interface ISession {
     sessionData?: InboundGroupSessionData;
 }
 
+/** Extended data on a Megolm session */
+export interface SessionExtended extends ISession {
+    needsBackup: boolean;
+}
+
 /** Data on an Olm session */
 export interface ISessionInfo {
     deviceKey?: string;

src/crypto/store/indexeddb-crypto-store-backend.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -815,29 +816,39 @@ export class Backend implements CryptoStore {
      *
      * Implementation of {@link CryptoStore.getEndToEndInboundGroupSessionsBatch}.
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | ISession[]> {
-        const result: ISession[] = [];
-        await this.doTxn("readonly", [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
-            const objectStore = txn.objectStore(IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS);
-            const getReq = objectStore.openCursor();
-            getReq.onsuccess = function (): void {
-                try {
-                    const cursor = getReq.result;
-                    if (cursor) {
-                        result.push({
-                            senderKey: cursor.value.senderCurve25519Key,
-                            sessionId: cursor.value.sessionId,
-                            sessionData: cursor.value.session,
-                        });
-                        if (result.length < SESSION_BATCH_SIZE) {
-                            cursor.continue();
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | SessionExtended[]> {
+        const result: SessionExtended[] = [];
+        await this.doTxn(
+            "readonly",
+            [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS, IndexedDBCryptoStore.STORE_BACKUP],
+            (txn) => {
+                const sessionStore = txn.objectStore(IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS);
+                const backupStore = txn.objectStore(IndexedDBCryptoStore.STORE_BACKUP);
+
+                const getReq = sessionStore.openCursor();
+                getReq.onsuccess = function (): void {
+                    try {
+                        const cursor = getReq.result;
+                        if (cursor) {
+                            const backupGetReq = backupStore.get(cursor.key);
+                            backupGetReq.onsuccess = (): void => {
+                                result.push({
+                                    senderKey: cursor.value.senderCurve25519Key,
+                                    sessionId: cursor.value.sessionId,
+                                    sessionData: cursor.value.session,
+                                    needsBackup: backupGetReq.result !== undefined,
+                                });
+                                if (result.length < SESSION_BATCH_SIZE) {
+                                    cursor.continue();
+                                }
+                            };
                         }
+                    } catch (e) {
+                        abortWithException(txn, <Error>e);
                     }
-                } catch (e) {
-                    abortWithException(txn, <Error>e);
-                }
-            };
-        });
+                };
+            },
+        );
 
         if (result.length === 0) {
             // No sessions left.

src/crypto/store/indexeddb-crypto-store.ts

@@ -25,6 +25,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -610,7 +611,7 @@ export class IndexedDBCryptoStore implements CryptoStore {
      *
      * @internal
      */
-    public getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null> {
+    public getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null> {
         return this.backend!.getEndToEndInboundGroupSessionsBatch();
     }
 

src/crypto/store/localStorage-crypto-store.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -357,20 +358,24 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore implements Crypto
      *
      * @internal
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<ISession[] | null> {
-        const result: ISession[] = [];
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<SessionExtended[] | null> {
+        const sessionsNeedingBackup = getJsonItem<string[]>(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        const result: SessionExtended[] = [];
         for (let i = 0; i < this.store.length; ++i) {
             const key = this.store.key(i);
             if (key?.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                const key2 = key.slice(KEY_INBOUND_SESSION_PREFIX.length);
+
                 // we can't use split, as the components we are trying to split out
                 // might themselves contain '/' characters. We rely on the
                 // senderKey being a (32-byte) curve25519 key, base64-encoded
                 // (hence 43 characters long).
 
                 result.push({
-                    senderKey: key.slice(KEY_INBOUND_SESSION_PREFIX.length, KEY_INBOUND_SESSION_PREFIX.length + 43),
-                    sessionId: key.slice(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    senderKey: key2.slice(0, 43),
+                    sessionId: key2.slice(44),
                     sessionData: getJsonItem(this.store, key)!,
+                    needsBackup: key2 in sessionsNeedingBackup,
                 });
 
                 if (result.length >= SESSION_BATCH_SIZE) {

src/crypto/store/memory-crypto-store.ts

@@ -21,6 +21,7 @@ import {
     IDeviceData,
     IProblem,
     ISession,
+    SessionExtended,
     ISessionInfo,
     IWithheld,
     MigrationState,
@@ -534,13 +535,14 @@ export class MemoryCryptoStore implements CryptoStore {
      *
      * @internal
      */
-    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | ISession[]> {
-        const result: ISession[] = [];
+    public async getEndToEndInboundGroupSessionsBatch(): Promise<null | SessionExtended[]> {
+        const result: SessionExtended[] = [];
         for (const [key, session] of Object.entries(this.inboundGroupSessions)) {
             result.push({
                 senderKey: key.slice(0, 43),
                 sessionId: key.slice(44),
                 sessionData: session,
+                needsBackup: key in this.sessionsNeedingBackup,
             });
             if (result.length >= SESSION_BATCH_SIZE) {
                 return result;

src/rust-crypto/index.ts

@@ -15,12 +15,15 @@ limitations under the License.
 */
 
 import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+import { StoreHandle } from "@matrix-org/matrix-sdk-crypto-wasm";
 
 import { RustCrypto } from "./rust-crypto";
 import { IHttpOpts, MatrixHttpApi } from "../http-api";
 import { ServerSideSecretStorage } from "../secret-storage";
 import { ICryptoCallbacks } from "../crypto";
 import { Logger } from "../logger";
+import { CryptoStore } from "../crypto/store/base";
+import { migrateFromLegacyCrypto } from "./libolm_migration";
 
 /**
  * Create a new `RustCrypto` implementation
@@ -63,6 +66,12 @@ export async function initRustCrypto(args: {
      * will be unencrypted.
      */
     storePassphrase?: string;
+
+    /** If defined, we will check if any data needs migrating from this store to the rust store. */
+    legacyCryptoStore?: CryptoStore;
+
+    /** The pickle key for `legacyCryptoStore` */
+    legacyPickleKey?: string;
 }): Promise<RustCrypto> {
     const { logger } = args;
 
@@ -73,6 +82,21 @@ export async function initRustCrypto(args: {
     // enable tracing in the rust-sdk
     new RustSdkCryptoJs.Tracing(RustSdkCryptoJs.LoggerLevel.Debug).turnOn();
 
+    logger.debug("Opening Rust CryptoStore");
+    const storeHandle: StoreHandle = await StoreHandle.open(
+        args.storePrefix ?? undefined,
+        (args.storePrefix && args.storePassphrase) ?? undefined,
+    );
+
+    if (args.legacyCryptoStore) {
+        // We have a legacy crypto store, which we may need to migrate from.
+        await migrateFromLegacyCrypto({
+            legacyStore: args.legacyCryptoStore,
+            storeHandle,
+            ...args,
+        });
+    }
+
     const rustCrypto = await initOlmMachine(
         logger,
         args.http,
@@ -80,10 +104,11 @@ export async function initRustCrypto(args: {
         args.deviceId,
         args.secretStorage,
         args.cryptoCallbacks,
-        args.storePrefix,
-        args.storePassphrase,
+        storeHandle,
     );
 
+    storeHandle.free();
+
     logger.debug("Completed rust crypto-sdk setup");
     return rustCrypto;
 }
@@ -95,15 +120,14 @@ async function initOlmMachine(
     deviceId: string,
     secretStorage: ServerSideSecretStorage,
     cryptoCallbacks: ICryptoCallbacks,
-    storePrefix: string | null,
-    storePassphrase: string | undefined,
+    storeHandle: StoreHandle,
 ): Promise<RustCrypto> {
     logger.debug("Init OlmMachine");
-    const olmMachine = await RustSdkCryptoJs.OlmMachine.initialize(
+
+    const olmMachine = await RustSdkCryptoJs.OlmMachine.init_from_store(
         new RustSdkCryptoJs.UserId(userId),
         new RustSdkCryptoJs.DeviceId(deviceId),
-        storePrefix ?? undefined,
-        (storePrefix && storePassphrase) ?? undefined,
+        storeHandle,
     );
 
     // Disable room key requests, per https://github.com/vector-im/element-web/issues/26524.

src/rust-crypto/libolm_migration.ts

@@ -0,0 +1,228 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
+
+import { Logger } from "../logger";
+import { CryptoStore, MigrationState, SecretStorePrivateKeys } from "../crypto/store/base";
+import { IndexedDBCryptoStore } from "../crypto/store/indexeddb-crypto-store";
+import { decryptAES, IEncryptedPayload } from "../crypto/aes";
+import { IHttpOpts, MatrixHttpApi } from "../http-api";
+import { requestKeyBackupVersion } from "./backup";
+
+/**
+ * Determine if any data needs migrating from the legacy store, and do so.
+ *
+ * @param args - Arguments object.
+ */
+export async function migrateFromLegacyCrypto(args: {
+    /** A `Logger` instance that will be used for debug output. */
+    logger: Logger;
+
+    /**
+     * Low-level HTTP interface: used to make outgoing requests required by the rust SDK.
+     * We expect it to set the access token, etc.
+     */
+    http: MatrixHttpApi<IHttpOpts & { onlyData: true }>;
+
+    /** Store to migrate data from. */
+    legacyStore: CryptoStore;
+
+    /** Pickle key for `legacyStore`. */
+    legacyPickleKey?: string;
+
+    /** Local user's User ID. */
+    userId: string;
+
+    /** Local user's Device ID. */
+    deviceId: string;
+
+    /** Rust crypto store to migrate data into. */
+    storeHandle: RustSdkCryptoJs.StoreHandle;
+}): Promise<void> {
+    const { logger, legacyStore } = args;
+
+    // initialise the rust matrix-sdk-crypto-wasm, if it hasn't already been done
+    await RustSdkCryptoJs.initAsync();
+
+    // enable tracing in the rust-sdk
+    new RustSdkCryptoJs.Tracing(RustSdkCryptoJs.LoggerLevel.Debug).turnOn();
+
+    if (!(await legacyStore.containsData())) {
+        // This store was never used. Nothing to migrate.
+        return;
+    }
+
+    await legacyStore.startup();
+    let migrationState = await legacyStore.getMigrationState();
+
+    if (migrationState === MigrationState.MEGOLM_SESSIONS_MIGRATED) {
+        // All migration is done.
+        return;
+    }
+
+    const pickleKey = new TextEncoder().encode(args.legacyPickleKey);
+
+    if (migrationState === MigrationState.NOT_STARTED) {
+        logger.info("Migrating data from legacy crypto store. Step 1: base data");
+        await migrateBaseData(args.http, args.userId, args.deviceId, legacyStore, pickleKey, args.storeHandle);
+
+        migrationState = MigrationState.INITIAL_DATA_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+
+    if (migrationState === MigrationState.INITIAL_DATA_MIGRATED) {
+        logger.info("Migrating data from legacy crypto store. Step 2: olm sessions");
+        await migrateOlmSessions(logger, legacyStore, pickleKey, args.storeHandle);
+
+        migrationState = MigrationState.OLM_SESSIONS_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+
+    if (migrationState === MigrationState.OLM_SESSIONS_MIGRATED) {
+        logger.info("Migrating data from legacy crypto store. Step 3: megolm sessions");
+        await migrateMegolmSessions(logger, legacyStore, pickleKey, args.storeHandle);
+
+        migrationState = MigrationState.MEGOLM_SESSIONS_MIGRATED;
+        await legacyStore.setMigrationState(migrationState);
+    }
+
+    logger.info("Migration from legacy crypto store complete");
+}
+
+async function migrateBaseData(
+    http: MatrixHttpApi<IHttpOpts & { onlyData: true }>,
+    userId: string,
+    deviceId: string,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+): Promise<void> {
+    const migrationData = new RustSdkCryptoJs.BaseMigrationData();
+    migrationData.userId = new RustSdkCryptoJs.UserId(userId);
+    migrationData.deviceId = new RustSdkCryptoJs.DeviceId(deviceId);
+
+    await legacyStore.doTxn("readonly", [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) =>
+        legacyStore.getAccount(txn, (a) => {
+            migrationData.pickledAccount = a ?? "";
+        }),
+    );
+
+    const recoveryKey = await getAndDecryptCachedSecretKey(legacyStore, pickleKey, "m.megolm_backup.v1");
+
+    // If we have a backup recovery key, we need to try to figure out which backup version it is for.
+    // All we can really do is ask the server for the most recent version.
+    if (recoveryKey) {
+        const backupInfo = await requestKeyBackupVersion(http);
+        if (backupInfo) {
+            migrationData.backupVersion = backupInfo.version;
+            migrationData.backupRecoveryKey = recoveryKey;
+        }
+    }
+
+    migrationData.privateCrossSigningMasterKey = await getAndDecryptCachedSecretKey(legacyStore, pickleKey, "master");
+    migrationData.privateCrossSigningSelfSigningKey = await getAndDecryptCachedSecretKey(
+        legacyStore,
+        pickleKey,
+        "self_signing",
+    );
+    migrationData.privateCrossSigningUserSigningKey = await getAndDecryptCachedSecretKey(
+        legacyStore,
+        pickleKey,
+        "user_signing",
+    );
+    await RustSdkCryptoJs.Migration.migrateBaseData(migrationData, pickleKey, storeHandle);
+}
+
+async function migrateOlmSessions(
+    logger: Logger,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const batch = await legacyStore.getEndToEndSessionsBatch();
+        if (batch === null) return;
+
+        logger.debug(`Migrating batch of ${batch.length} olm sessions`);
+        const migrationData: RustSdkCryptoJs.PickledSession[] = [];
+        for (const session of batch) {
+            const pickledSession = new RustSdkCryptoJs.PickledSession();
+            pickledSession.senderKey = session.deviceKey!;
+            pickledSession.pickle = session.session!;
+            pickledSession.lastUseTime = pickledSession.creationTime = new Date(session.lastReceivedMessageTs!);
+            migrationData.push(pickledSession);
+        }
+
+        await RustSdkCryptoJs.Migration.migrateOlmSessions(migrationData, pickleKey, storeHandle);
+        await legacyStore.deleteEndToEndSessionsBatch(batch);
+    }
+}
+
+async function migrateMegolmSessions(
+    logger: Logger,
+    legacyStore: CryptoStore,
+    pickleKey: Uint8Array,
+    storeHandle: RustSdkCryptoJs.StoreHandle,
+): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const batch = await legacyStore.getEndToEndInboundGroupSessionsBatch();
+        if (batch === null) return;
+
+        logger.debug(`Migrating batch of ${batch.length} megolm sessions`);
+        const migrationData: RustSdkCryptoJs.PickledInboundGroupSession[] = [];
+        for (const session of batch) {
+            const pickledSession = new RustSdkCryptoJs.PickledInboundGroupSession();
+            pickledSession.pickle = session.sessionData!.session;
+            pickledSession.roomId = new RustSdkCryptoJs.RoomId(session.sessionData!.room_id);
+            pickledSession.senderKey = session.senderKey;
+            pickledSession.senderSigningKey = session.sessionData!.keysClaimed["ed25519"];
+            pickledSession.backedUp = !session.needsBackup;
+
+            // Not sure if we can reliably distinguish imported vs not-imported sessions in the libolm database.
+            // For now at least, let's be conservative and say that all the sessions are imported (which means that
+            // the Rust SDK treats them as less secure).
+            pickledSession.imported = true;
+
+            migrationData.push(pickledSession);
+        }
+
+        await RustSdkCryptoJs.Migration.migrateMegolmSessions(migrationData, pickleKey, storeHandle);
+        await legacyStore.deleteEndToEndInboundGroupSessionsBatch(batch);
+    }
+}
+
+async function getAndDecryptCachedSecretKey(
+    legacyStore: CryptoStore,
+    legacyPickleKey: Uint8Array,
+    name: string,
+): Promise<string | undefined> {
+    let encodedKey: IEncryptedPayload | null = null;
+
+    await legacyStore.doTxn("readonly", "account", (txn) => {
+        legacyStore.getSecretStorePrivateKey(
+            txn,
+            (k) => {
+                encodedKey = k as IEncryptedPayload | null;
+            },
+            name as keyof SecretStorePrivateKeys,
+        );
+    });
+
+    return encodedKey === null ? undefined : await decryptAES(encodedKey, legacyPickleKey, name);
+}