fix indexedDB storage and retry decryption when we get an olm error

src/crypto/algorithms/base.js

@@ -159,6 +159,16 @@ class DecryptionAlgorithm {
     shareKeysWithDevice(keyRequest) {
         throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
     }
+
+    /**
+     * Retry decrypting all the events from a sender that haven't been
+     * decrypted yet.
+     *
+     * @param {string} senderKey the sender's key
+     */
+    async retryDecryptionFromSender(senderKey) {
+        // ignore by default
+    }
 }
 export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 

src/crypto/algorithms/megolm.js

@@ -997,15 +997,17 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
         // scheduled, so we needn't send out the request here.)
         this._requestKeysForEvent(event);
 
+        // See if there was a problem with the olm session at the time the
+        // event was sent.  Use a fuzz factor of 2 minutes.
         const problem = await this._olmDevice.sessionMayHaveProblems(
-            content.sender_key, event.getTs(),
+            content.sender_key, event.getTs() - 120000,
         );
         if (problem) {
             let problemDescription = PROBLEM_DESCRIPTIONS[problem.type]
                   || PROBLEM_DESCRIPTIONS.unknown;
             if (problem.fixed) {
                 problemDescription +=
-                    " Trying to create a new secure channel and re-requesting the keys";
+                    " Trying to create a new secure channel and re-requesting the keys.";
             }
             throw new base.DecryptionError(
                 "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
@@ -1071,11 +1073,16 @@ MegolmDecryption.prototype._requestKeysForEvent = function(event) {
  */
 MegolmDecryption.prototype._addEventToPendingList = function(event) {
     const content = event.getWireContent();
-    const k = content.sender_key + "|" + content.session_id;
-    if (!this._pendingEvents[k]) {
-        this._pendingEvents[k] = new Set();
+    const senderKey = content.sender_key;
+    const sessionId = content.session_id;
+    if (!this._pendingEvents[senderKey]) {
+        this._pendingEvents[senderKey] = new Map();
     }
-    this._pendingEvents[k].add(event);
+    const senderPendingEvents = this._pendingEvents[senderKey];
+    if (!senderPendingEvents.has(sessionId)) {
+        senderPendingEvents.set(sessionId, new Set());
+    }
+    senderPendingEvents.get(sessionId).add(event);
 };
 
 /**
@@ -1087,14 +1094,20 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
  */
 MegolmDecryption.prototype._removeEventFromPendingList = function(event) {
     const content = event.getWireContent();
-    const k = content.sender_key + "|" + content.session_id;
-    if (!this._pendingEvents[k]) {
+    const senderKey = content.sender_key;
+    const sessionId = content.session_id;
+    const senderPendingEvents = this._pendingEvents[senderKey];
+    const pendingEvents = senderPendingEvents && senderPendingEvents.get(sessionId);
+    if (!pendingEvents) {
         return;
     }
 
-    this._pendingEvents[k].delete(event);
-    if (this._pendingEvents[k].size === 0) {
-        delete this._pendingEvents[k];
+    pendingEvents.delete(event);
+    if (pendingEvents.size === 0) {
+        senderPendingEvents.delete(senderKey);
+    }
+    if (senderPendingEvents.size === 0) {
+        delete this._pendingEvents[senderKey];
     }
 };
 
@@ -1211,10 +1224,17 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
         const sender = event.getSender();
         // if the sender says that they haven't been able to establish an olm
         // session, let's proactively establish one
+
+        // Note: after we record that the olm session has had a problem, we
+        // trigger retrying decryption for all the messages from the sender's
+        // key, so that we can update the error message to indicate the olm
+        // session problem.
+
         if (await this._olmDevice.getSessionIdForDevice(senderKey)) {
             // a session has already been established, so we don't need to
             // create a new one.
             await this._olmDevice.recordSessionProblem(senderKey, "no_olm", true);
+            this.retryDecryptionFromSender(senderKey);
             return;
         }
         const device = this._crypto._deviceList.getDeviceByIdentityKey(
@@ -1226,6 +1246,7 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
                     ": not establishing session",
             );
             await this._olmDevice.recordSessionProblem(senderKey, "no_olm", false);
+            this.retryDecryptionFromSender(senderKey);
             return;
         }
         await olmlib.ensureOlmSessionsForDevices(
@@ -1247,6 +1268,7 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
         );
 
         await this._olmDevice.recordSessionProblem(senderKey, "no_olm", true);
+        this.retryDecryptionFromSender(senderKey);
 
         await this._baseApis.sendToDevice("m.room.encrypted", {
             [sender]: {
@@ -1404,13 +1426,20 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
  * @return {Boolean} whether all messages were successfully decrypted
  */
 MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionId) {
-    const k = senderKey + "|" + sessionId;
-    const pending = this._pendingEvents[k];
+    const senderPendingEvents = this._pendingEvents[senderKey];
+    if (!senderPendingEvents) {
+        return true;
+    }
+
+    const pending = senderPendingEvents.get(sessionId);
     if (!pending) {
         return true;
     }
 
-    delete this._pendingEvents[k];
+    pending.delete(sessionId);
+    if (pending.size === 0) {
+        this._pendingEvents[senderKey];
+    }
 
     await Promise.all([...pending].map(async (ev) => {
         try {
@@ -1420,7 +1449,32 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
         }
     }));
 
-    return !this._pendingEvents[k];
+    // ev.attemptDecryption will re-add to this._pendingEvents if an event
+    // couldn't be decrypted
+    return !((this._pendingEvents[senderKey] || {})[sessionId]);
+};
+
+MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey) {
+    const senderPendingEvents = this._pendingEvents[senderKey];
+    logger.warn(senderPendingEvents);
+    if (!senderPendingEvents) {
+        return true;
+    }
+
+    delete this._pendingEvents[senderKey];
+
+    await Promise.all([...senderPendingEvents].map(async ([_sessionId, pending]) => {
+        await Promise.all([...pending].map(async (ev) => {
+            try {
+                logger.warn(ev.getId());
+                await ev.attemptDecryption(this._crypto);
+            } catch (e) {
+                // don't die if something goes wrong
+            }
+        }));
+    }));
+
+    return !this._pendingEvents[senderKey];
 };
 
 base.registerAlgorithm(

src/crypto/index.js

@@ -2464,10 +2464,25 @@ Crypto.prototype._onRoomKeyWithheldEvent = function(event) {
         return;
     }
 
+    logger.info(
+        `Got room key withheld event from ${event.getSender()} (${content.sender_key}) `
+            + `for ${content.algorithm}/${content.room_id}/${content.session_id} `
+            + `with reason ${content.code} (${content.reason})`,
+    );
+
     const alg = this._getRoomDecryptor(content.room_id, content.algorithm);
     if (alg.onRoomKeyWithheldEvent) {
         alg.onRoomKeyWithheldEvent(event);
     }
+    if (!content.room_id) {
+        // retry decryption for all events sent by the sender_key.  This will
+        // update the events to show a message indicating that the olm session was
+        // wedged.
+        const roomDecryptors = this._getRoomDecryptors(content.algorithm);
+        for (const decryptor of roomDecryptors) {
+            decryptor.retryDecryptionFromSender(content.sender_key);
+        }
+    }
 };
 
 /**
@@ -2583,6 +2598,16 @@ Crypto.prototype._onToDeviceBadEncrypted = async function(event) {
     const algorithm = content.algorithm;
     const deviceKey = content.sender_key;
 
+    // retry decryption for all events sent by the sender_key.  This will
+    // update the events to show a message indicating that the olm session was
+    // wedged.
+    const retryDecryption = () => {
+        const roomDecryptors = this._getRoomDecryptors(olmlib.MEGOLM_ALGORITHM);
+        for (const decryptor of roomDecryptors) {
+            decryptor.retryDecryptionFromSender(deviceKey);
+        }
+    };
+
     if (sender === undefined || deviceKey === undefined || deviceKey === undefined) {
         return;
     }
@@ -2596,6 +2621,8 @@ Crypto.prototype._onToDeviceBadEncrypted = async function(event) {
             "New session already forced with device " + sender + ":" + deviceKey +
             " at " + lastNewSessionForced + ": not forcing another",
         );
+        await this._olmDevice.recordSessionProblem(deviceKey, "wedged", true);
+        retryDecryption();
         return;
     }
 
@@ -2609,9 +2636,8 @@ Crypto.prototype._onToDeviceBadEncrypted = async function(event) {
             "Couldn't find device for identity key " + deviceKey +
             ": not re-establishing session",
         );
-        await this._olmDevice.recordSessionProblem(
-            deviceKey, "wedged", false,
-        );
+        await this._olmDevice.recordSessionProblem(deviceKey, "wedged", false);
+        retryDecryption();
         return;
     }
     const devicesByUser = {};
@@ -2644,6 +2670,7 @@ Crypto.prototype._onToDeviceBadEncrypted = async function(event) {
     );
 
     await this._olmDevice.recordSessionProblem(deviceKey, "wedged", true);
+    retryDecryption();
 
     await this._baseApis.sendToDevice("m.room.encrypted", {
         [sender]: {
@@ -2926,6 +2953,24 @@ Crypto.prototype._getRoomDecryptor = function(roomId, algorithm) {
 };
 
 
+/**
+ * Get all the room decryptors for a given encryption algorithm.
+ *
+ * @param {string} algorithm The encryption algorithm
+ *
+ * @return {array} An array of room decryptors
+ */
+Crypto.prototype._getRoomDecryptors = function(algorithm) {
+    const decryptors = [];
+    for (const d of Object.values(this._roomDecryptors)) {
+        if (algorithm in d) {
+            decryptors.push(d[algorithm]);
+        }
+    }
+    return decryptors;
+};
+
+
 /**
  * sign the given object with our ed25519 key
  *

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -450,6 +450,9 @@ export class Backend {
                 result = null;
                 return;
             }
+            problems.sort((a, b) => {
+                return a.time - b.time;
+            });
             const lastProblem = problems[problems.length - 1];
             for (const problem of problems) {
                 if (problem.time > timestamp) {
@@ -768,7 +771,7 @@ export function upgradeDatabase(db, oldVersion) {
         const problemsStore = db.createObjectStore("session_problems", {
             keyPath: ["deviceKey", "time"],
         });
-        problemsStore.createIndex("deviceKey");
+        problemsStore.createIndex("deviceKey", "deviceKey");
 
         db.createObjectStore("notified_error_devices", {
             keyPath: ["userId", "deviceId"],

src/crypto/store/indexeddb-crypto-store.js

@@ -417,7 +417,7 @@ export default class IndexedDBCryptoStore {
 
     getEndToEndSessionProblem(deviceKey, timestamp) {
         return this._backendPromise.then(async (backend) => {
-            await backend.getEndToEndSessionProblem(deviceKey, timestamp);
+            return await backend.getEndToEndSessionProblem(deviceKey, timestamp);
         });
     }