Merge remote-tracking branch 'origin/develop' into revert-1493-revert-1487-dbkr/tsify_call

package.json

@@ -84,7 +84,7 @@
     "jest-localstorage-mock": "^2.4.3",
     "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
     "rimraf": "^3.0.2",
     "terser": "^4.8.0",
     "tsify": "^4.0.2",

spec/browserify/sync-browserify.spec.js

@@ -99,5 +99,5 @@ describe("Browserify Test", function() {
                 client.once("sync.unexpectedError", reject);
             }),
         ]);
-    }, 10000);
+    }, 20000); // additional timeout as this test can take quite a while
 });

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -84,9 +84,16 @@ describe("CrossSigningInfo.getCrossSigningKey", function() {
         const info = new CrossSigningInfo(userId, {
             getCrossSigningKey: () => testKey,
         });
-        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
         expect(pubKey).toEqual(masterKeyPub);
-        expect(ab).toEqual({a: 106712, b: 106712});
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
     });
 
     it.each(types)("should request a key from the cache callback (if set)" +

src/@types/global.d.ts

@@ -14,7 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import * as Olm from "olm";
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
 
 export {};
 
@@ -22,12 +23,8 @@ declare global {
     namespace NodeJS {
         interface Global {
             localStorage: Storage;
-            Olm: Olm;
         }
     }
-    interface Global {
-        Olm: Olm;
-    }
 
     interface MediaDevices {
         // This is experimental and types don't know about it yet

src/client.js

@@ -5595,9 +5595,8 @@ function _PojoToMatrixEventMapper(client, options) {
             }
             event.attemptDecryption(client._crypto);
         }
-        const room = client.getRoom(event.getRoomId());
+        if (!preventReEmit) {
-        if (room && !preventReEmit) {
+            client.reEmitter.reEmit(event, ["Event.replaced"]);
-            room.reEmitter.reEmit(event, ["Event.replaced"]);
         }
         return event;
     }

src/crypto/OlmDevice.js

@@ -477,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *

src/crypto/index.js

@@ -1859,6 +1859,14 @@ Crypto.prototype.updateOneTimeKeyCount = function(currentCount) {
     }
 };
 
+Crypto.prototype.setNeedsNewFallback = function(needsNewFallback) {
+    this._needsNewFallback = !!needsNewFallback;
+};
+
+Crypto.prototype.getNeedsNewFallback = function() {
+    return this._needsNewFallback;
+};
+
 // check if it's time to upload one-time keys, and do so if so.
 function _maybeUploadOneTimeKeys(crypto) {
     // frequency with which to check & upload one-time keys
@@ -1906,27 +1914,31 @@ function _maybeUploadOneTimeKeys(crypto) {
     // out stale private keys that won't receive a message.
     const keyLimit = Math.floor(maxOneTimeKeys / 2);
 
-    function uploadLoop(keyCount) {
+    async function uploadLoop(keyCount) {
-        if (keyLimit <= keyCount) {
+        while (keyLimit > keyCount || crypto.getNeedsNewFallback()) {
-            // If we don't need to generate any more keys then we are done.
+            // Ask olm to generate new one time keys, then upload them to synapse.
-            return Promise.resolve();
+            if (keyLimit > keyCount) {
+                logger.info("generating oneTimeKeys");
+                const keysThisLoop = Math.min(keyLimit - keyCount, maxKeysPerCycle);
+                await crypto._olmDevice.generateOneTimeKeys(keysThisLoop);
             }
 
-        const keysThisLoop = Math.min(keyLimit - keyCount, maxKeysPerCycle);
+            if (crypto.getNeedsNewFallback()) {
+                logger.info("generating fallback key");
+                await crypto._olmDevice.generateFallbackKey();
+            }
 
-        // Ask olm to generate new one time keys, then upload them to synapse.
+            logger.info("calling _uploadOneTimeKeys");
-        return crypto._olmDevice.generateOneTimeKeys(keysThisLoop).then(() => {
+            const res = await _uploadOneTimeKeys(crypto);
-            return _uploadOneTimeKeys(crypto);
-        }).then((res) => {
             if (res.one_time_key_counts && res.one_time_key_counts.signed_curve25519) {
                 // if the response contains a more up to date value use this
                 // for the next loop
-                return uploadLoop(res.one_time_key_counts.signed_curve25519);
+                keyCount = res.one_time_key_counts.signed_curve25519;
             } else {
-                throw new Error("response for uploading keys does not contain "
+                throw new Error("response for uploading keys does not contain " +
-                              + "one_time_key_counts.signed_curve25519");
+                                "one_time_key_counts.signed_curve25519");
+            }
         }
-        });
     }
 
     crypto._oneTimeKeyCheckInProgress = true;
@@ -1958,11 +1970,22 @@ function _maybeUploadOneTimeKeys(crypto) {
 
 // returns a promise which resolves to the response
 async function _uploadOneTimeKeys(crypto) {
+    const promises = [];
+
+    const fallbackJson = {};
+    if (crypto.getNeedsNewFallback()) {
+        const fallbackKeys = await crypto._olmDevice.getFallbackKey();
+        for (const [keyId, key] of Object.entries(fallbackKeys.curve25519)) {
+            const k = { key, fallback: true };
+            fallbackJson["signed_curve25519:" + keyId] = k;
+            promises.push(crypto._signObject(k));
+        }
+        crypto.setNeedsNewFallback(false);
+    }
+
     const oneTimeKeys = await crypto._olmDevice.getOneTimeKeys();
     const oneTimeJson = {};
 
-    const promises = [];
-
     for (const keyId in oneTimeKeys.curve25519) {
         if (oneTimeKeys.curve25519.hasOwnProperty(keyId)) {
             const k = {
@@ -1976,7 +1999,8 @@ async function _uploadOneTimeKeys(crypto) {
     await Promise.all(promises);
 
     const res = await crypto._baseApis.uploadKeysRequest({
-        one_time_keys: oneTimeJson,
+        "one_time_keys": oneTimeJson,
+        "org.matrix.msc2732.fallback_keys": fallbackJson,
     });
 
     await crypto._olmDevice.markKeysAsPublished();

src/models/event-timeline-set.js

@@ -806,7 +806,7 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
             this.room,
         );
         isNewRelations = true;
-        relatesToEvent = this.findEventById(relatesToEventId);
+        relatesToEvent = this.findEventById(relatesToEventId) || this.room.getPendingEvent(relatesToEventId);
         if (relatesToEvent) {
             relationsWithEventType.setTargetEvent(relatesToEvent);
         }

src/models/event.js

@@ -904,6 +904,8 @@ utils.extend(MatrixEvent.prototype, {
     /**
      * Set an event that replaces the content of this event, through an m.replace relation.
      *
+     * @fires module:models/event.MatrixEvent#"Event.replaced"
+     *
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {

src/models/room.js

@@ -373,6 +373,20 @@ Room.prototype.hasPendingEvent = function(eventId) {
     return this._pendingEventList.some(event => event.getId() === eventId);
 };
 
+/**
+ * Get a specific event from the pending event list, if configured, null otherwise.
+ *
+ * @param {string} eventId The event ID to check for.
+ * @return {MatrixEvent}
+ */
+Room.prototype.getPendingEvent = function(eventId) {
+    if (this._opts.pendingEventOrdering !== "detached") {
+        return null;
+    }
+
+    return this._pendingEventList.find(event => event.getId() === eventId);
+};
+
 /**
  * Get the live unfiltered timeline for this room.
  *

src/sync.js

@@ -1361,6 +1361,16 @@ SyncApi.prototype._processSyncResponse = async function(
         const currentCount = data.device_one_time_keys_count.signed_curve25519 || 0;
         this.opts.crypto.updateOneTimeKeyCount(currentCount);
     }
+    if (this.opts.crypto && data["org.matrix.msc2732.device_unused_fallback_key_types"]) {
+        // The presence of device_unused_fallback_key_types indicates that the
+        // server supports fallback keys. If there's no unused
+        // signed_curve25519 fallback key we need a new one.
+        const unusedFallbackKeys = data["org.matrix.msc2732.device_unused_fallback_key_types"];
+        this.opts.crypto.setNeedsNewFallback(
+            unusedFallbackKeys instanceof Array &&
+            !unusedFallbackKeys.includes("signed_curve25519"),
+        );
+    }
 };
 
 /**

yarn.lock

@@ -5411,9 +5411,9 @@ object.values@^1.1.1:
     function-bind "^1.1.1"
     has "^1.0.3"
 
-"olm@https://packages.matrix.org/npm/olm/olm-3.1.4.tgz":
+"olm@https://packages.matrix.org/npm/olm/olm-3.2.1.tgz":
-  version "3.1.4"
+  version "3.2.1"
-  resolved "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz#0f03128b7d3b2f614d2216409a1dfccca765fdb3"
+  resolved "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz#d623d76f99c3518dde68be8c86618d68bc7b004a"
 
 once@^1.3.0, once@^1.3.1, once@^1.4.0:
   version "1.4.0"