Reorganize key backup flow

This will allow the key backup flow to propagate errors for things like version
mismatches more easily.

In addition, it raises the limit of keys sent per request from 10 to 200 to cut
down on the number of requests.

src/crypto/index.js

@@ -42,6 +42,7 @@ export function isCryptoAvailable() {
 }
 
 const MIN_FORCE_SESSION_INTERVAL_MS = 60 * 60 * 1000;
+const KEY_BACKUP_KEYS_PER_REQUEST = 200;
 
 /**
  * Cryptography bits
@@ -986,29 +987,69 @@ Crypto.prototype.importRoomKeys = function(keys) {
     );
 };
 
-Crypto.prototype._maybeSendKeyBackup = async function(delay, retry) {
-    if (retry === undefined) retry = true;
+/**
+ * Schedules sending all keys waiting to be sent to the backup, if not already
+ * scheduled. Retries if necessary.
+ */
+Crypto.prototype._scheduleKeyBackupSend = async function() {
+    if (this._sendingBackups) return;
 
-    if (!this._sendingBackups) {
-        this._sendingBackups = true;
     try {
-            if (delay === undefined) {
-                // by default, wait between 0 and 10 seconds, to avoid backup
+        // wait between 0 and 10 seconds, to avoid backup
         // requests from different clients hitting the server all at
         // the same time when a new key is sent
-                delay = Math.random() * 10000;
-            }
+        const delay = Math.random() * 10000;
         await Promise.delay(delay);
         let numFailures = 0; // number of consecutive failures
         while (1) {
             if (!this.backupKey) {
                 return;
             }
-                // FIXME: figure out what limit is reasonable
-                const sessions = await this._cryptoStore.getSessionsNeedingBackup(10);
-                if (!sessions.length) {
+            try {
+                const numBackedUp =
+                    await this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST);
+                if (numBackedUp === 0) {
+                    // no sessions left needing backup: we're done
                     return;
                 }
+                numFailures = 0;
+            } catch (err) {
+                numFailures++;
+                console.log("Key backup request failed", err);
+                if (err.data) {
+                    if (
+                        err.data.errcode == 'M_NOT_FOUND' ||
+                        err.data.errcode == 'M_WRONG_ROOM_KEYS_VERSION'
+                    ) {
+                        // Backup version has changed or this backup version
+                        // has been deleted
+                        throw err;
+                    }
+                }
+            }
+            if (numFailures) {
+                // exponential backoff if we have failures
+                await Promise.delay(1000 * Math.pow(2, Math.min(numFailures - 1, 4)));
+            }
+        }
+    } finally {
+        this._sendingBackups = false;
+    }
+};
+
+/**
+ * Take some e2e keys waiting to be backed up and send them
+ * to the backup.
+ *
+ * @param {integer} limit Maximum number of keys to back up
+ * @returns {integer} Number of sessions backed up
+ */
+Crypto.prototype._backupPendingKeys = async function(limit) {
+    const sessions = await this._cryptoStore.getSessionsNeedingBackup(limit);
+    if (!sessions.length) {
+        return 0;
+    }
+
     const data = {};
     for (const session of sessions) {
         const roomId = session.sessionData.room_id;
@@ -1041,39 +1082,13 @@ Crypto.prototype._maybeSendKeyBackup = async function(delay, retry) {
         };
     }
 
-                try {
     await this._baseApis.sendKeyBackup(
         undefined, undefined, this.backupInfo.version,
         {rooms: data},
     );
-                    numFailures = 0;
     await this._cryptoStore.unmarkSessionsNeedingBackup(sessions);
-                } catch (err) {
-                    numFailures++;
-                    console.log("send failed", err);
-                    if (err.httpStatus === 400
-                        || err.httpStatus === 403
-                        || err.httpStatus === 401
-                        || !retry) {
-                        // retrying probably won't help much, so we should give up
-                        // FIXME: disable backups?
-                        throw err;
-                    }
-                }
-                if (numFailures) {
-                    // exponential backoff if we have failures
-                    await new Promise((resolve, reject) => {
-                        setTimeout(
-                            resolve,
-                            1000 * Math.pow(2, Math.min(numFailures - 1, 4)),
-                        );
-                    });
-                }
-            }
-        } finally {
-            this._sendingBackups = false;
-        }
-    }
+
+    return sessions.length;
 };
 
 Crypto.prototype.backupGroupSession = async function(
@@ -1090,7 +1105,9 @@ Crypto.prototype.backupGroupSession = async function(
         sessionId: sessionId,
     }]);
 
-    await this._maybeSendKeyBackup();
+    // don't wait for this to complete: it will delay so
+    // happens in the background
+    this._scheduleKeyBackupSend();
 };
 
 Crypto.prototype.backupAllGroupSessions = async function(version) {
@@ -1109,7 +1126,10 @@ Crypto.prototype.backupAllGroupSessions = async function(version) {
         },
     );
 
-    await this._maybeSendKeyBackup(0, false);
+    let numKeysBackedUp;
+    do {
+        numKeysBackedUp = this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST);
+    } while (numKeysBackedUp > 0);
 };
 
 /* eslint-disable valid-jsdoc */    //https://github.com/eslint/eslint/issues/7307