wrap backup sending in a try, and add delays

spec/unit/crypto/backup.spec.js

@@ -171,6 +171,7 @@ describe("MegolmBackup", function() {
         });
 
         it('sends backups to the server', function () {
+            this.timeout(12000);
             const groupSession = new global.Olm.OutboundGroupSession();
             groupSession.create();
             const ibGroupSession = new global.Olm.InboundGroupSession();
@@ -263,6 +264,7 @@ describe("MegolmBackup", function() {
         });
 
         it('retries when a backup fails', function () {
+            this.timeout(12000);
             const groupSession = new global.Olm.OutboundGroupSession();
             groupSession.create();
             const ibGroupSession = new global.Olm.InboundGroupSession();

src/crypto/index.js

@@ -969,54 +969,68 @@ Crypto.prototype.importRoomKeys = function(keys) {
 Crypto.prototype._maybeSendKeyBackup = async function() {
     if (!this._sendingBackups) {
         this._sendingBackups = true;
-        while (1) {
+        try {
-            if (!this.backupKey) {
+            // wait between 0 and 10 seconds, to avoid backup requests from
-                this._sendingBackups = false;
+            // different clients hitting the server all at the same time when a
-                return;
+            // new key is sent
-            }
+            await new Promise((resolve, reject) => {
-            // FIXME: figure out what limit is reasonable
+                setTimeout(resolve, Math.random() * 10000);
-            const sessions = await this._cryptoStore.getSessionsNeedingBackup(10);
+            });
-            if (!sessions.length) {
+            let numFailures = 0; // number of consecutive failures
-                this._sendingBackups = false;
+            while (1) {
-                return;
-            }
-            const data = {};
-            for (const session of sessions) {
-                const room_id = session.sessionData.room_id;
-                if (data[room_id] === undefined)
-                    data[room_id] = {sessions: {}};
-
-                const sessionData = await this._olmDevice.exportInboundGroupSession(session.senderKey, session.sessionId, session.sessionData);
-                sessionData.algorithm = olmlib.MEGOLM_ALGORITHM;
-                delete sessionData.session_id;
-                delete sessionData.room_id;
-                const encrypted = this.backupKey.encrypt(JSON.stringify(sessionData));
-
-                data[room_id]['sessions'][session.sessionId] = {
-                    first_message_index: 1, // FIXME
-                    forwarded_count: (sessionData.forwardingCurve25519KeyChain || []).length,
-                    is_verified: false, // FIXME: how do we determine this?
-                    session_data: encrypted,
-                };
-            }
-
-            let successful = false;
-            do {
                 if (!this.backupKey) {
-                    this._sendingBackups = false;
                     return;
                 }
+                // FIXME: figure out what limit is reasonable
+                const sessions = await this._cryptoStore.getSessionsNeedingBackup(10);
+                if (!sessions.length) {
+                    return;
+                }
+                const data = {};
+                for (const session of sessions) {
+                    const room_id = session.sessionData.room_id;
+                    if (data[room_id] === undefined)
+                        data[room_id] = {sessions: {}};
+
+                    const sessionData = await this._olmDevice.exportInboundGroupSession(session.senderKey, session.sessionId, session.sessionData);
+                    sessionData.algorithm = olmlib.MEGOLM_ALGORITHM;
+                    delete sessionData.session_id;
+                    delete sessionData.room_id;
+                    const encrypted = this.backupKey.encrypt(JSON.stringify(sessionData));
+
+                    data[room_id]['sessions'][session.sessionId] = {
+                        first_message_index: 1, // FIXME
+                        forwarded_count: (sessionData.forwardingCurve25519KeyChain || []).length,
+                        is_verified: false, // FIXME: how do we determine this?
+                        session_data: encrypted,
+                    };
+                }
+
                 try {
                     await this._baseApis.sendKeyBackup(undefined, undefined, this.backupInfo.version, {rooms: data});
-                    successful = true;
+                    numFailures = 0;
                     await this._cryptoStore.unmarkSessionsNeedingBackup(sessions);
                 }
-                catch (e) {
+                catch (err) {
-                    console.log("send failed", e);
+                    numFailures++;
-                    // FIXME: pause
+                    console.log("send failed", err);
+                    if (err.httpStatus === 400 || err.httpStatus === 403 || err.httpStatus === 401) {
+                        // retrying probably won't help much, so we should give up
+                        // FIXME: disable backups?
+                        return;
+                    }
                 }
-            } while (!successful);
+                if (numFailures) {
-            // FIXME: pause between iterations?
+                    // exponential backoff if we have failures
+                    await new Promise((resolve, reject) => {
+                        setTimeout(resolve, 1000 * Math.pow(2, Math.min(numFailures - 1, 4)));
+                    });
+                }
+            }
+        }
+        finally
+        {
+            this._sendingBackups = false;
         }
     }
 }