[CONFLICT CHUNKS] Merge branch 'develop' into travis/sourcemaps-dev

spec/integ/megolm-integ.spec.js

@@ -615,6 +615,9 @@ describe("megolm", function() {
             ).respond(200, {
                 event_id: '$event_id',
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
@@ -712,6 +715,9 @@ describe("megolm", function() {
                     event_id: '$event_id',
                 };
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test2'),

spec/unit/crypto/algorithms/megolm.spec.js

@@ -8,6 +8,22 @@ import {Crypto} from "../../../../src/crypto";
 import {logger} from "../../../../src/logger";
 import {MatrixEvent} from "../../../../src/models/event";
 
+<<<<<<< HEAD
+=======
+import sdk from '../../../..';
+import algorithms from '../../../../lib/crypto/algorithms';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+import Crypto from '../../../../lib/crypto';
+import logger from '../../../../lib/logger';
+import TestClient from '../../../TestClient';
+import olmlib from '../../../../lib/crypto/olmlib';
+import Room from '../../../../lib/models/room';
+
+const MatrixEvent = sdk.MatrixEvent;
+>>>>>>> develop
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 const MegolmEncryption = algorithms.ENCRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 
@@ -340,4 +356,154 @@ describe("MegolmDecryption", function() {
             expect(ct2.session_id).toEqual(ct1.session_id);
         });
     });
+
+    it("notifies devices that have been blocked", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient1 = (new TestClient(
+            "@bob:example.com", "bobdevice1",
+        )).client;
+        const bobClient2 = (new TestClient(
+            "@bob:example.com", "bobdevice2",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient1.initCrypto(),
+            bobClient2.initCrypto(),
+        ]);
+        const aliceDevice = aliceClient._crypto._olmDevice;
+        const bobDevice1 = bobClient1._crypto._olmDevice;
+        const bobDevice2 = bobClient2._crypto._olmDevice;
+
+        const encryptionCfg = {
+            "algorithm": "m.megolm.v1.aes-sha2",
+        };
+        const roomId = "!someroom";
+        const room = new Room(roomId, aliceClient, "@alice:example.com", {});
+        room.getEncryptionTargetMembers = async function() {
+            return [{userId: "@bob:example.com"}];
+        };
+        room.setBlacklistUnverifiedDevices(true);
+        aliceClient.store.storeRoom(room);
+        await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+
+        const BOB_DEVICES = {
+            bobdevice1: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice1",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice1.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice1.deviceCurve25519Key,
+                },
+                verified: 0,
+            },
+            bobdevice2: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice2",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice2.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice2.deviceCurve25519Key,
+                },
+                verified: -1,
+            },
+        };
+
+        aliceClient._crypto._deviceList.storeDevicesForUser(
+            "@bob:example.com", BOB_DEVICES,
+        );
+        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+            return this._getDevicesFromStore(userIds);
+        };
+
+        let run = false;
+        aliceClient.sendToDevice = async (msgtype, contentMap) => {
+            run = true;
+            expect(msgtype).toBe("org.matrix.room_key.withheld");
+            delete contentMap["@bob:example.com"].bobdevice1.session_id;
+            delete contentMap["@bob:example.com"].bobdevice2.session_id;
+            expect(contentMap).toStrictEqual({
+                '@bob:example.com': {
+                    bobdevice1: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.unverified',
+                        reason:
+                        'The sender has disabled encrypting to unverified devices.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                    bobdevice2: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.blacklisted',
+                        reason: 'The sender has blocked you.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                },
+            });
+        };
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            sender: "@alice:example.com",
+            room_id: roomId,
+            event_id: "$event",
+            content: {
+                msgtype: "m.text",
+                body: "secret",
+            },
+        });
+        await aliceClient._crypto.encryptEvent(event, room);
+
+        expect(run).toBe(true);
+
+        aliceClient.stopClient();
+        bobClient1.stopClient();
+        bobClient2.stopClient();
+    });
+
+    it("throws an error describing why it doesn't have a key", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const roomId = "!someroom";
+
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "org.matrix.room_key.withheld",
+            sender: "@bob:example.com",
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                room_id: roomId,
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+                code: "m.blacklisted",
+                reason: "You have been blocked",
+            },
+        }));
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+        }))).rejects.toThrow("The sender has blocked you.");
+    });
 });

spec/unit/crypto/backup.spec.js

@@ -334,7 +334,7 @@ describe("MegolmBackup", function() {
             });
             await client.resetCrossSigningKeys();
             let numCalls = 0;
-            await new Promise(async (resolve, reject) => {
+            await new Promise((resolve, reject) => {
                 client._http.authedRequest = function(
                     callback, method, path, queryParams, data, opts,
                 ) {
@@ -359,7 +359,7 @@ describe("MegolmBackup", function() {
                     resolve();
                     return Promise.resolve({});
                 };
-                await client.createKeyBackupVersion({
+                client.createKeyBackupVersion({
                     algorithm: "m.megolm_backup.v1",
                     auth_data: {
                         public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",

src/client.js

@@ -22,6 +22,7 @@ limitations under the License.
  * This is an internal module. See {@link MatrixClient} for the public class.
  * @module client
  */
+<<<<<<< HEAD
 
 import url from "url";
 import {EventEmitter} from "events";
@@ -47,6 +48,37 @@ import {decodeRecoveryKey} from './crypto/recoverykey';
 import {keyFromAuthData} from './crypto/key_passphrase';
 import {randomString} from './randomstring';
 import {PushProcessor} from "./pushprocessor";
+=======
+const EventEmitter = require("events").EventEmitter;
+const url = require('url');
+
+const httpApi = require("./http-api");
+const MatrixEvent = require("./models/event").MatrixEvent;
+const EventStatus = require("./models/event").EventStatus;
+const EventTimeline = require("./models/event-timeline");
+const SearchResult = require("./models/search-result");
+const StubStore = require("./store/stub");
+const webRtcCall = require("./webrtc/call");
+const utils = require("./utils");
+const contentRepo = require("./content-repo");
+const Filter = require("./filter");
+const SyncApi = require("./sync");
+const MatrixBaseApis = require("./base-apis");
+const MatrixError = httpApi.MatrixError;
+const ContentHelpers = require("./content-helpers");
+const olmlib = require("./crypto/olmlib");
+
+import ReEmitter from './ReEmitter';
+import RoomList from './crypto/RoomList';
+import logger from './logger';
+
+import Crypto from './crypto';
+import { isCryptoAvailable } from './crypto';
+import { decodeRecoveryKey } from './crypto/recoverykey';
+import { keyFromAuthData } from './crypto/key_passphrase';
+import { randomString } from './randomstring';
+import { encodeBase64, decodeBase64 } from '../lib/crypto/olmlib';
+>>>>>>> develop
 
 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -666,7 +698,7 @@ MatrixClient.prototype.initCrypto = async function() {
         "crypto.warning",
         "crypto.devicesUpdated",
         "deviceVerificationChanged",
-        "userVerificationChanged",
+        "userTrustStatusChanged",
         "crossSigning.keysChanged",
     ]);
 
@@ -1431,7 +1463,7 @@ MatrixClient.prototype.disableKeyBackup = function() {
  *     when restoring the backup as an alternative to entering the recovery key.
  *     Optional.
  * @param {boolean} [opts.secureSecretStorage = false] Whether to use Secure
- *     Secret Storage (MSC1946) to store the key encrypting key backups.
+ *     Secret Storage to store the key encrypting key backups.
  *     Optional, defaults to false.
  *
  * @returns {Promise<object>} Object that can be passed to createKeyBackupVersion and
@@ -1445,32 +1477,37 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
         throw new Error("End-to-end encryption disabled");
     }
 
-    if (secureSecretStorage) {
+    const [keyInfo, encodedPrivateKey, privateKey] =
-        logger.log("Preparing key backup version with Secure Secret Storage");
-
-        // Ensure Secure Secret Storage is ready for use
-        if (!this.hasSecretStorageKey()) {
-            throw new Error("Secure Secret Storage has no keys, needs bootstrapping");
-        }
-
-        throw new Error("Not yet implemented");
-    }
-
-    const [keyInfo, encodedPrivateKey] =
         await this.createRecoveryKeyFromPassphrase(password);
 
+    if (secureSecretStorage) {
+        await this.storeSecret("m.megolm_backup.v1", encodeBase64(privateKey));
+        logger.info("Key backup private key stored in secret storage");
+    }
+
     // Reshape objects into form expected for key backup
+    const authData = {
+        public_key: keyInfo.pubkey,
+    };
+    if (keyInfo.passphrase) {
+        authData.private_key_salt = keyInfo.passphrase.salt;
+        authData.private_key_iterations = keyInfo.passphrase.iterations;
+    }
     return {
         algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
-        auth_data: {
+        auth_data: authData,
-            public_key: keyInfo.pubkey,
-            private_key_salt: keyInfo.passphrase.salt,
-            private_key_iterations: keyInfo.passphrase.iterations,
-        },
         recovery_key: encodedPrivateKey,
     };
 };
 
+/**
+ * Check whether the key backup private key is stored in secret storage.
+ * @return {Promise<boolean>} Whether the backup key is stored.
+ */
+MatrixClient.prototype.isKeyBackupKeyStored = async function() {
+    return this.isSecretStored("m.megolm_backup.v1", false /* checkKey */);
+};
+
 /**
  * Create a new key backup version and enable it, using the information return
  * from prepareKeyBackupVersion.
@@ -1488,13 +1525,19 @@ MatrixClient.prototype.createKeyBackupVersion = async function(info) {
         auth_data: info.auth_data,
     };
 
-    // Now sign the backup auth data. Do it as this device first because crypto._signObject
+    // Sign the backup auth data with the device key for backwards compat with
-    // is dumb and bluntly replaces the whole signatures block...
+    // older devices with cross-signing. This can probably go away very soon in
-    // this can probably go away very soon in favour of just signing with the SSK.
+    // favour of just signing with the cross-singing master key.
     await this._crypto._signObject(data.auth_data);
 
-    if (this._crypto._crossSigningInfo.getId()) {
+    if (
-        // now also sign the auth data with the master key
+        this._cryptoCallbacks.getCrossSigningKey &&
+        this._crypto._crossSigningInfo.getId()
+    ) {
+        // now also sign the auth data with the cross-signing master key
+        // we check for the callback explicitly here because we still want to be able
+        // to create an un-cross-signed key backup if there is a cross-signing key but
+        // no callback supplied.
         await this._crypto._crossSigningInfo.signObject(data.auth_data, "master");
     }
 
@@ -1502,11 +1545,15 @@ MatrixClient.prototype.createKeyBackupVersion = async function(info) {
         undefined, "POST", "/room_keys/version", undefined, data,
         {prefix: PREFIX_UNSTABLE},
     );
-    this.enableKeyBackup({
+
-        algorithm: info.algorithm,
+    // We could assume everything's okay and enable directly, but this ensures
-        auth_data: info.auth_data,
+    // we run the same signature verification that will be used for future
-        version: res.version,
+    // sessions.
-    });
+    await this.checkKeyBackup();
+    if (!this.getKeyBackupEnabled()) {
+        logger.error("Key backup not usable even though we just created it");
+    }
+
     return res;
 };
 
@@ -1610,6 +1657,18 @@ MatrixClient.prototype.isValidRecoveryKey = function(recoveryKey) {
 
 MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
 
+/**
+ * Restore from an existing key backup via a passphrase.
+ *
+ * @param {string} password Passphrase
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
 MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
     password, targetRoomId, targetSessionId, backupInfo,
 ) {
@@ -1619,6 +1678,39 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
     );
 };
 
+/**
+ * Restore from an existing key backup via a private key stored in secret
+ * storage.
+ *
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
+MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
+    backupInfo, targetRoomId, targetSessionId,
+) {
+    const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
+    return this._restoreKeyBackup(
+        privKey, targetRoomId, targetSessionId, backupInfo,
+    );
+};
+
+/**
+ * Restore from an existing key backup via an encoded recovery key.
+ *
+ * @param {string} recoveryKey Encoded recovery key
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
 MatrixClient.prototype.restoreKeyBackupWithRecoveryKey = function(
     recoveryKey, targetRoomId, targetSessionId, backupInfo,
 ) {

src/crypto/OlmDevice.js

@@ -1,7 +1,11 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017, 2019 New Vector Ltd
+<<<<<<< HEAD
 Copyright 2019 The Matrix.org Foundation C.I.C.
+=======
+Copyright 2020 The Matrix.org Foundation C.I.C.
+>>>>>>> develop
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,6 +23,8 @@ limitations under the License.
 import {logger} from '../logger';
 import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
 
+import algorithms from './algorithms';
+
 // The maximum size of an event is 65K, and we base64 the content, so this is a
 // reasonable approximation to the biggest plaintext we can encrypt.
 const MAX_PLAINTEXT_LENGTH = 65536 * 3 / 4;
@@ -819,9 +825,9 @@ OlmDevice.prototype._getInboundGroupSession = function(
     roomId, senderKey, sessionId, txn, func,
 ) {
     this._cryptoStore.getEndToEndInboundGroupSession(
-        senderKey, sessionId, txn, (sessionData) => {
+        senderKey, sessionId, txn, (sessionData, withheld) => {
             if (sessionData === null) {
-                func(null);
+                func(null, null, withheld);
                 return;
             }
 
@@ -835,7 +841,7 @@ OlmDevice.prototype._getInboundGroupSession = function(
             }
 
             this._unpickleInboundGroupSession(sessionData, (session) => {
-                func(session, sessionData);
+                func(session, sessionData, withheld);
             });
         },
     );
@@ -860,7 +866,10 @@ OlmDevice.prototype.addInboundGroupSession = async function(
     exportFormat,
 ) {
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             /* if we already have this session, consider updating it */
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn,
@@ -915,6 +924,60 @@ OlmDevice.prototype.addInboundGroupSession = async function(
     );
 };
 
+/**
+ * Record in the data store why an inbound group session was withheld.
+ *
+ * @param {string} roomId     room that the session belongs to
+ * @param {string} senderKey  base64-encoded curve25519 key of the sender
+ * @param {string} sessionId  session identifier
+ * @param {string} code       reason code
+ * @param {string} reason     human-readable version of `code`
+ */
+OlmDevice.prototype.addInboundGroupSessionWithheld = async function(
+    roomId, senderKey, sessionId, code, reason,
+) {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD],
+        (txn) => {
+            this._cryptoStore.storeEndToEndInboundGroupSessionWithheld(
+                senderKey, sessionId,
+                {
+                    room_id: roomId,
+                    code: code,
+                    reason: reason,
+                },
+                txn,
+            );
+        },
+    );
+};
+
+export const WITHHELD_MESSAGES = {
+    "m.unverified": "The sender has disabled encrypting to unverified devices.",
+    "m.blacklisted": "The sender has blocked you.",
+    "m.unauthorised": "You are not authorised to read the message.",
+    "m.no_olm": "Unable to establish a secure channel.",
+};
+
+/**
+ * Calculate the message to use for the exception when a session key is withheld.
+ *
+ * @param {object} withheld  An object that describes why the key was withheld.
+ *
+ * @return {string} the message
+ *
+ * @private
+ */
+function _calculateWithheldMessage(withheld) {
+    if (withheld.code && withheld.code in WITHHELD_MESSAGES) {
+        return WITHHELD_MESSAGES[withheld.code];
+    } else if (withheld.reason) {
+        return withheld.reason;
+    } else {
+        return "decryption key withheld";
+    }
+}
+
 /**
  * Decrypt a received message with an inbound group session
  *
@@ -935,16 +998,49 @@ OlmDevice.prototype.decryptGroupMessage = async function(
     roomId, senderKey, sessionId, body, eventId, timestamp,
 ) {
     let result;
+    // when the localstorage crypto store is used as an indexeddb backend,
+    // exceptions thrown from within the inner function are not passed through
+    // to the top level, so we store exceptions in a variable and raise them at
+    // the end
+    let error;
 
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
-                roomId, senderKey, sessionId, txn, (session, sessionData) => {
+                roomId, senderKey, sessionId, txn, (session, sessionData, withheld) => {
                     if (session === null) {
+                        if (withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        }
                         result = null;
                         return;
                     }
-                    const res = session.decrypt(body);
+                    let res;
+                    try {
+                        res = session.decrypt(body);
+                    } catch (e) {
+                        if (e && e.message === 'OLM.UNKNOWN_MESSAGE_INDEX' && withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        } else {
+                            error = e;
+                        }
+                        return;
+                    }
 
                     let plaintext = res.plaintext;
                     if (plaintext === undefined) {
@@ -966,10 +1062,11 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                                 msgInfo.id !== eventId ||
                                 msgInfo.timestamp !== timestamp
                             ) {
-                                throw new Error(
+                                error = new Error(
                                     "Duplicate message index, possible replay attack: " +
                                     messageIndexKey,
                                 );
+                                return;
                             }
                         }
                         this._inboundGroupSessionMessageIndexes[messageIndexKey] = {
@@ -995,6 +1092,9 @@ OlmDevice.prototype.decryptGroupMessage = async function(
         },
     );
 
+    if (error) {
+        throw error;
+    }
     return result;
 };
 
@@ -1010,7 +1110,10 @@ OlmDevice.prototype.decryptGroupMessage = async function(
 OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, sessionId) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._cryptoStore.getEndToEndInboundGroupSession(
                 senderKey, sessionId, txn, (sessionData) => {
                     if (sessionData === null) {
@@ -1061,7 +1164,10 @@ OlmDevice.prototype.getInboundGroupSessionKey = async function(
 ) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn, (session, sessionData) => {
                     if (session === null) {

src/crypto/SecretStorage.js

@@ -231,6 +231,27 @@ export class SecretStorage extends EventEmitter {
         await this._baseApis.setAccountData(name, {encrypted});
     }
 
+    /**
+     * Store a secret defined to be the same as the given key.
+     * No secret information will be stored, instead the secret will
+     * be stored with a marker to say that the contents of the secret is
+     * the value of the given key.
+     * This is useful for migration from systems that predate SSSS such as
+     * key backup.
+     *
+     * @param {string} name The name of the secret
+     * @param {string} keyId The ID of the key whose value will be the
+     *     value of the secret
+     * @returns {Promise} resolved when account data is saved
+     */
+    storePassthrough(name, keyId) {
+        return this._baseApis.setAccountData(name, {
+            [keyId]: {
+                passthrough: true,
+            },
+        });
+    }
+
     /**
      * Get a secret from storage.
      *
@@ -276,8 +297,13 @@ export class SecretStorage extends EventEmitter {
             // fetch private key from app
             [keyId, decryption] = await this._getSecretStorageKey(keys);
 
-            // decrypt secret
             const encInfo = secretContent.encrypted[keyId];
+
+            // We don't actually need the decryption object if it's a passthrough
+            // since we just want to return the key itself.
+            if (encInfo.passthrough) return decryption.get_private_key();
+
+            // decrypt secret
             switch (keys[keyId].algorithm) {
             case SECRET_STORAGE_ALGORITHM_V1:
                 return decryption.decrypt(

src/crypto/algorithms/megolm.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,6 +34,8 @@ import {
     UnknownDeviceError,
 } from "./base";
 
+import {WITHHELD_MESSAGES} from '../OlmDevice';
+
 /**
  * @private
  * @constructor
@@ -52,6 +55,7 @@ function OutboundSessionInfo(sessionId) {
     this.useCount = 0;
     this.creationTime = new Date().getTime();
     this.sharedWithDevices = {};
+    this.blockedDevicesNotified = {};
 }
 
 
@@ -89,6 +93,15 @@ OutboundSessionInfo.prototype.markSharedWithDevice = function(
     this.sharedWithDevices[userId][deviceId] = chainIndex;
 };
 
+OutboundSessionInfo.prototype.markNotifiedBlockedDevice = function(
+    userId, deviceId,
+) {
+    if (!this.blockedDevicesNotified[userId]) {
+        this.blockedDevicesNotified[userId] = {};
+    }
+    this.blockedDevicesNotified[userId][deviceId] = true;
+};
+
 /**
  * Determine if this session has been shared with devices which it shouldn't
  * have been.
@@ -171,11 +184,14 @@ utils.inherits(MegolmEncryption, EncryptionAlgorithm);
  * @private
  *
  * @param {Object} devicesInRoom The devices in this room, indexed by user ID
+ * @param {Object} blocked The devices that are blocked, indexed by user ID
  *
  * @return {module:client.Promise} Promise which resolves to the
  *    OutboundSessionInfo when setup is complete.
  */
-MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
+MegolmEncryption.prototype._ensureOutboundSession = async function(
+    devicesInRoom, blocked,
+) {
     const self = this;
 
     let session;
@@ -242,9 +258,36 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
             }
         }
 
-        return self._shareKeyWithDevices(
+        await self._shareKeyWithDevices(
             session, shareMap,
         );
+
+        // are there any new blocked devices that we need to notify?
+        const blockedMap = {};
+        for (const userId in blocked) {
+            if (!blocked.hasOwnProperty(userId)) {
+                continue;
+            }
+
+            const userBlockedDevices = blocked[userId];
+
+            for (const deviceId in userBlockedDevices) {
+                if (!userBlockedDevices.hasOwnProperty(deviceId)) {
+                    continue;
+                }
+
+                if (
+                    !session.blockedDevicesNotified[userId] ||
+                        session.blockedDevicesNotified[userId][deviceId] === undefined
+                ) {
+                    blockedMap[userId] = blockedMap[userId] || [];
+                    blockedMap[userId].push(userBlockedDevices[deviceId]);
+                }
+            }
+        }
+
+        // notify blocked devices that they're blocked
+        await self._notifyBlockedDevices(session, blockedMap);
     }
 
     // helper which returns the session prepared by prepareSession
@@ -368,6 +411,42 @@ MegolmEncryption.prototype._splitUserDeviceMap = function(
     return mapSlices;
 };
 
+/**
+ * @private
+ *
+ * @param {object} devicesByUser map from userid to list of devices
+ *
+ * @return {array<array<object>>} the blocked devices, split into chunks
+ */
+MegolmEncryption.prototype._splitBlockedDevices = function(devicesByUser) {
+    const maxToDeviceMessagesPerRequest = 20;
+
+    // use an array where the slices of a content map gets stored
+    let currentSlice = [];
+    const mapSlices = [currentSlice];
+
+    for (const userId of Object.keys(devicesByUser)) {
+        const userBlockedDevicesToShareWith = devicesByUser[userId];
+
+        for (const blockedInfo of userBlockedDevicesToShareWith) {
+            if (currentSlice.length > maxToDeviceMessagesPerRequest) {
+                // the current slice is filled up. Start inserting into the next slice
+                currentSlice = [];
+                mapSlices.push(currentSlice);
+            }
+
+            currentSlice.push({
+                userId: userId,
+                blockedInfo: blockedInfo,
+            });
+        }
+    }
+    if (currentSlice.length === 0) {
+        mapSlices.pop();
+    }
+    return mapSlices;
+};
+
 /**
  * @private
  *
@@ -432,6 +511,49 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
     });
 };
 
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {array<object>} userDeviceMap list of blocked devices to notify
+ *
+ * @param {object} payload fields to include in the notification payload
+ *
+ * @return {module:client.Promise} Promise which resolves once the notifications
+ *     for the given userDeviceMap is generated and has been sent.
+ */
+MegolmEncryption.prototype._sendBlockedNotificationsToDevices = async function(
+    session, userDeviceMap, payload,
+) {
+    const contentMap = {};
+
+    for (const val of userDeviceMap) {
+        const userId = val.userId;
+        const blockedInfo = val.blockedInfo;
+        const deviceInfo = blockedInfo.deviceInfo;
+        const deviceId = deviceInfo.deviceId;
+
+        const message = Object.assign({}, payload);
+        message.code = blockedInfo.code;
+        message.reason = blockedInfo.reason;
+
+        if (!contentMap[userId]) {
+            contentMap[userId] = {};
+        }
+        contentMap[userId][deviceId] = message;
+    }
+
+    await this._baseApis.sendToDevice("org.matrix.room_key.withheld", contentMap);
+
+    // store that we successfully uploaded the keys of the current slice
+    for (const userId of Object.keys(contentMap)) {
+        for (const deviceId of Object.keys(contentMap[userId])) {
+            session.markNotifiedBlockedDevice(userId, deviceId);
+        }
+    }
+};
+
 /**
  * Re-shares a megolm session key with devices if the key has already been
  * sent to them.
@@ -566,6 +688,42 @@ MegolmEncryption.prototype._shareKeyWithDevices = async function(session, device
     }
 };
 
+/**
+ * Notify blocked devices that they have been blocked.
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object<string, object>} devicesByUser
+ *    map from userid to device ID to blocked data
+ */
+MegolmEncryption.prototype._notifyBlockedDevices = async function(
+    session, devicesByUser,
+) {
+    const payload = {
+        room_id: this._roomId,
+        session_id: session.sessionId,
+        algorithm: olmlib.MEGOLM_ALGORITHM,
+        sender_key: this._olmDevice.deviceCurve25519Key,
+    };
+
+    const userDeviceMaps = this._splitBlockedDevices(devicesByUser);
+
+    for (let i = 0; i < userDeviceMaps.length; i++) {
+        try {
+            await this._sendBlockedNotificationsToDevices(
+                session, userDeviceMaps[i], payload,
+            );
+            logger.log(`Completed blacklist notification for ${session.sessionId} `
+                + `in ${this._roomId} (slice ${i + 1}/${userDeviceMaps.length})`);
+        } catch (e) {
+            logger.log(`blacklist notification for ${session.sessionId} in `
+                + `${this._roomId} (slice ${i + 1}/${userDeviceMaps.length}) failed`);
+
+            throw e;
+        }
+    }
+};
+
 /**
  * @inheritdoc
  *
@@ -575,42 +733,41 @@ MegolmEncryption.prototype._shareKeyWithDevices = async function(session, device
  *
  * @return {module:client.Promise} Promise which resolves to the new event body
  */
-MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
+MegolmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
     const self = this;
     logger.log(`Starting to encrypt event for ${this._roomId}`);
 
-    return this._getDevicesInRoom(room).then(function(devicesInRoom) {
+    const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
-        // check if any of these devices are not yet known to the user.
-        // if so, warn the user so they can verify or ignore.
-        self._checkForUnknownDevices(devicesInRoom);
 
-        return self._ensureOutboundSession(devicesInRoom);
+    // check if any of these devices are not yet known to the user.
-    }).then(function(session) {
+    // if so, warn the user so they can verify or ignore.
-        const payloadJson = {
+    self._checkForUnknownDevices(devicesInRoom);
-            room_id: self._roomId,
-            type: eventType,
-            content: content,
-        };
 
-        const ciphertext = self._olmDevice.encryptGroupMessage(
+    const session = await self._ensureOutboundSession(devicesInRoom, blocked);
-            session.sessionId, JSON.stringify(payloadJson),
+    const payloadJson = {
-        );
+        room_id: self._roomId,
+        type: eventType,
+        content: content,
+    };
 
-        const encryptedContent = {
+    const ciphertext = self._olmDevice.encryptGroupMessage(
-            algorithm: olmlib.MEGOLM_ALGORITHM,
+        session.sessionId, JSON.stringify(payloadJson),
-            sender_key: self._olmDevice.deviceCurve25519Key,
+    );
-            ciphertext: ciphertext,
-            session_id: session.sessionId,
-             // Include our device ID so that recipients can send us a
-             // m.new_device message if they don't have our session key.
-             // XXX: Do we still need this now that m.new_device messages
-             // no longer exist since #483?
-            device_id: self._deviceId,
-        };
 
-        session.useCount++;
+    const encryptedContent = {
-        return encryptedContent;
+        algorithm: olmlib.MEGOLM_ALGORITHM,
-    });
+        sender_key: self._olmDevice.deviceCurve25519Key,
+        ciphertext: ciphertext,
+        session_id: session.sessionId,
+        // Include our device ID so that recipients can send us a
+        // m.new_device message if they don't have our session key.
+        // XXX: Do we still need this now that m.new_device messages
+        // no longer exist since #483?
+        device_id: self._deviceId,
+    };
+
+    session.useCount++;
+    return encryptedContent;
 };
 
 /**
@@ -659,8 +816,11 @@ MegolmEncryption.prototype._checkForUnknownDevices = function(devicesInRoom) {
  *
  * @param {module:models/room} room
  *
- * @return {module:client.Promise} Promise which resolves to a map
+ * @return {module:client.Promise} Promise which resolves to an array whose
- *     from userId to deviceId to deviceInfo
+ *     first element is a map from userId to deviceId to deviceInfo indicating
+ *     the devices that messages should be encrypted to, and whose second
+ *     element is a map from userId to deviceId to data indicating the devices
+ *     that are in the room but that have been blocked
  */
 MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     const members = await room.getEncryptionTargetMembers();
@@ -681,6 +841,7 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
     // using all the device_lists changes and left fields.
     // See https://github.com/vector-im/riot-web/issues/2305 for details.
     const devices = await this._crypto.downloadKeys(roomMembers, false);
+    const blocked = {};
     // remove any blocked devices
     for (const userId in devices) {
         if (!devices.hasOwnProperty(userId)) {
@@ -695,13 +856,27 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
 
             if (userDevices[deviceId].isBlocked() ||
                 (userDevices[deviceId].isUnverified() && isBlacklisting)
-               ) {
+            ) {
+                if (!blocked[userId]) {
+                    blocked[userId] = {};
+                }
+                const blockedInfo = userDevices[deviceId].isBlocked()
+                    ? {
+                        code: "m.blacklisted",
+                        reason: WITHHELD_MESSAGES["m.blacklisted"],
+                    }
+                    : {
+                        code: "m.unverified",
+                        reason: WITHHELD_MESSAGES["m.unverified"],
+                    };
+                blockedInfo.deviceInfo = userDevices[deviceId];
+                blocked[userId][deviceId] = blockedInfo;
                 delete userDevices[deviceId];
             }
         }
     }
 
-    return devices;
+    return [devices, blocked];
 };
 
 /**
@@ -761,6 +936,11 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
             event.getId(), event.getTs(),
         );
     } catch (e) {
+        if (e.name === "DecryptionError") {
+            // re-throw decryption errors as-is
+            throw e;
+        }
+
         let errorCode = "OLM_DECRYPT_GROUP_MESSAGE_ERROR";
 
         if (e && e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
@@ -968,6 +1148,20 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
     });
 };
 
+/**
+ * @inheritdoc
+ *
+ * @param {module:models/event.MatrixEvent} event key event
+ */
+MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
+    const content = event.getContent();
+
+    await this._olmDevice.addInboundGroupSessionWithheld(
+        content.room_id, content.sender_key, content.session_id, content.code,
+        content.reason,
+    );
+};
+
 /**
  * @inheritdoc
  */

src/crypto/index.js

@@ -2,7 +2,7 @@
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
 Copyright 2018-2019 New Vector Ltd
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019-2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -214,7 +214,7 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     );
 
     // Assuming no app-supplied callback, default to getting from SSSS.
-    if (!cryptoCallbacks.getCrossSigningKey) {
+    if (!cryptoCallbacks.getCrossSigningKey && cryptoCallbacks.getSecretStorageKey) {
         cryptoCallbacks.getCrossSigningKey = async (type) => {
             return CrossSigningInfo.getFromSecretStorage(type, this._secretStorage);
         };
@@ -292,8 +292,9 @@ Crypto.prototype.init = async function() {
  * @param {string} password Passphrase string that can be entered by the user
  *     when restoring the backup as an alternative to entering the recovery key.
  *     Optional.
- * @returns {Promise<Array>} Array with public key metadata and encoded private
+ * @returns {Promise<Array>} Array with public key metadata, encoded private
- *     recovery key which should be disposed of after displaying to the user.
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
  */
 Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
     const decryption = new global.Olm.PkDecryption();
@@ -310,10 +311,11 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
         } else {
             keyInfo.pubkey = decryption.generate_key();
         }
-        const encodedPrivateKey = encodeRecoveryKey(decryption.get_private_key());
+        const privateKey = decryption.get_private_key();
-        return [keyInfo, encodedPrivateKey];
+        const encodedPrivateKey = encodeRecoveryKey(privateKey);
+        return [keyInfo, encodedPrivateKey, privateKey];
     } finally {
-        decryption.free();
+        if (decryption) decryption.free();
     }
 };
 
@@ -330,6 +332,8 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
  *     auth data as an object.
  * @param {function} [opts.createSecretStorageKey] Optional. Function
  * called to await a secret storage key creation flow.
+ * @param {object} [opts.keyBackupInfo] The current key backup object. If passed,
+ * the passphrase and recovery key from this backup will be used.
  * Returns:
  *     {Promise} A promise which resolves to key creation data for
  *     SecretStorage#addKey: an object with `passphrase` and/or `pubkey` fields.
@@ -337,6 +341,7 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
 Crypto.prototype.bootstrapSecretStorage = async function({
     authUploadDeviceSigningKeys,
     createSecretStorageKey = async () => { },
+    keyBackupInfo,
 } = {}) {
     logger.log("Bootstrapping Secure Secret Storage");
 
@@ -377,18 +382,49 @@ Crypto.prototype.bootstrapSecretStorage = async function({
                     { authUploadDeviceSigningKeys },
                 );
             }
+        } else {
+            logger.log("Cross signing keys are present in secret storage");
         }
 
         // Check if Secure Secret Storage has a default key. If we don't have one, create
         // the default key (which will also be signed by the cross-signing master key).
         if (!this.hasSecretStorageKey()) {
-            logger.log("Secret storage default key not found, creating new key");
+            let newKeyId;
-            const keyOptions = await createSecretStorageKey();
+            if (keyBackupInfo) {
-            const newKeyId = await this.addSecretStorageKey(
+                logger.log("Secret storage default key not found, using key backup key");
-                SECRET_STORAGE_ALGORITHM_V1,
+                const opts = {
-                keyOptions,
+                    pubkey: keyBackupInfo.auth_data.public_key,
-            );
+                };
+
+                if (
+                    keyBackupInfo.auth_data.private_key_salt &&
+                    keyBackupInfo.auth_data.private_key_iterations
+                ) {
+                    opts.passphrase = {
+                        algorithm: "m.pbkdf2",
+                        iterations: keyBackupInfo.auth_data.private_key_iterations,
+                        salt: keyBackupInfo.auth_data.private_key_salt,
+                    };
+                }
+
+                newKeyId = await this.addSecretStorageKey(
+                    SECRET_STORAGE_ALGORITHM_V1, opts,
+                );
+
+                // Add an entry for the backup key in SSSS as a 'passthrough' key
+                // (ie. the secret is the key itself).
+                this._secretStorage.storePassthrough('m.megolm_backup.v1', newKeyId);
+            } else {
+                logger.log("Secret storage default key not found, creating new key");
+                const keyOptions = await createSecretStorageKey();
+                newKeyId = await this.addSecretStorageKey(
+                    SECRET_STORAGE_ALGORITHM_V1,
+                    keyOptions,
+                );
+            }
             await this.setDefaultSecretStorageKeyId(newKeyId);
+        } else {
+            logger.log("Have secret storage key");
         }
 
         // If cross-signing keys were reset, store them in Secure Secret Storage.
@@ -548,8 +584,8 @@ Crypto.prototype.resetCrossSigningKeys = async function(level, {
 
 /**
  * Run various follow-up actions after cross-signing keys have changed locally
- * (either by resetting the keys for the account or bye getting them from secret
+ * (either by resetting the keys for the account or by getting them from secret
- * storaoge), such as signing the current device, upgrading device
+ * storage), such as signing the current device, upgrading device
  * verifications, etc.
  */
 Crypto.prototype._afterCrossSigningLocalKeyChange = async function() {
@@ -784,7 +820,7 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
                 throw new Error("Cross-signing master private key not available");
             }
         } finally {
-            signing.free();
+            if (signing) signing.free();
         }
 
         logger.info("Got matching private key from callback for new public master key");
@@ -946,8 +982,7 @@ Crypto.prototype.setTrustedBackupPubKey = async function(trustedPubKey) {
  */
 Crypto.prototype.checkKeyBackup = async function() {
     this._checkedForBackup = false;
-    const returnInfo = await this._checkAndStartKeyBackup();
+    return this._checkAndStartKeyBackup();
-    return returnInfo;
 };
 
 /**
@@ -994,13 +1029,14 @@ Crypto.prototype.isKeyBackupTrusted = async function(backupInfo) {
             logger.log("Ignoring unknown signature type: " + keyIdParts[0]);
             continue;
         }
-        // Could be an SSK but just say this is the device ID for backwards compat
+        // Could be a cross-signing master key, but just say this is the device
-        const sigInfo = { deviceId: keyIdParts[1] }; // XXX: is this how we're supposed to get the device ID?
+        // ID for backwards compat
+        const sigInfo = { deviceId: keyIdParts[1] };
 
         // first check to see if it's from our cross-signing key
         const crossSigningId = this._crossSigningInfo.getId();
-        if (crossSigningId === keyId) {
+        if (crossSigningId === sigInfo.deviceId) {
-            sigInfo.cross_signing_key = crossSigningId;
+            sigInfo.crossSigningId = true;
             try {
                 await olmlib.verifySignature(
                     this._olmDevice,
@@ -1022,18 +1058,19 @@ Crypto.prototype.isKeyBackupTrusted = async function(backupInfo) {
 
         // Now look for a sig from a device
         // At some point this can probably go away and we'll just support
-        // it being signed by the SSK
+        // it being signed by the cross-signing master key
         const device = this._deviceList.getStoredDevice(
             this._userId, sigInfo.deviceId,
         );
         if (device) {
             sigInfo.device = device;
+            sigInfo.deviceTrust = await this.checkDeviceTrust(
+                this._userId, sigInfo.deviceId,
+            );
             try {
                 await olmlib.verifySignature(
                     this._olmDevice,
-                    // verifySignature modifies the object so we need to copy
+                    backupInfo.auth_data,
-                    // if we verify more than one sig
-                    Object.assign({}, backupInfo.auth_data),
                     this._userId,
                     device.deviceId,
                     device.getFingerprint(),
@@ -1057,8 +1094,8 @@ Crypto.prototype.isKeyBackupTrusted = async function(backupInfo) {
     ret.usable = ret.sigs.some((s) => {
         return (
             s.valid && (
-                (s.device && s.device.isVerified()) ||
+                (s.device && s.deviceTrust.isVerified()) ||
-                (s.cross_signing_key)
+                (s.crossSigningId)
             )
         );
     });
@@ -2367,6 +2404,8 @@ Crypto.prototype._onToDeviceEvent = function(event) {
             this._secretStorage._onRequestReceived(event);
         } else if (event.getType() === "m.secret.send") {
             this._secretStorage._onSecretReceived(event);
+        } else if (event.getType() === "org.matrix.room_key.withheld") {
+            this._onRoomKeyWithheldEvent(event);
         } else if (event.getContent().transaction_id) {
             this._onKeyVerificationMessage(event);
         } else if (event.getContent().msgtype === "m.bad.encrypted") {
@@ -2406,6 +2445,27 @@ Crypto.prototype._onRoomKeyEvent = function(event) {
     alg.onRoomKeyEvent(event);
 };
 
+/**
+ * Handle a key withheld event
+ *
+ * @private
+ * @param {module:models/event.MatrixEvent} event key withheld event
+ */
+Crypto.prototype._onRoomKeyWithheldEvent = function(event) {
+    const content = event.getContent();
+
+    if (!content.room_id || !content.session_id || !content.algorithm
+        || !content.sender_key) {
+        logger.error("key withheld event is missing fields");
+        return;
+    }
+
+    const alg = this._getRoomDecryptor(content.room_id, content.algorithm);
+    if (alg.onRoomKeyWithheldEvent) {
+        alg.onRoomKeyWithheldEvent(event);
+    }
+};
+
 /**
  * Handle a general key verification event.
  *

src/crypto/olmlib.js

@@ -302,8 +302,7 @@ async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceIn
  *
  * @param {module:crypto/OlmDevice} olmDevice olm wrapper to use for verify op
  *
- * @param {Object} obj object to check signature on. Note that this will be
+ * @param {Object} obj object to check signature on.
- * stripped of its 'signatures' and 'unsigned' properties.
  *
  * @param {string} signingUserId  ID of the user whose signature should be checked
  *

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,7 +19,7 @@ limitations under the License.
 import {logger} from '../../logger';
 import * as utils from "../../utils";
 
-export const VERSION = 7;
+export const VERSION = 8;
 
 /**
  * Implementation of a CryptoStore which is backed by an existing
@@ -79,7 +80,7 @@ export class Backend {
                     `enqueueing key request for ${requestBody.room_id} / ` +
                         requestBody.session_id,
                 );
-                txn.oncomplete = () => { resolve(request); };
+                txn.oncomplete = () => {resolve(request);};
                 const store = txn.objectStore("outgoingRoomKeyRequests");
                 store.add(request);
             });
@@ -428,14 +429,36 @@ export class Backend {
     // Inbound group sessions
 
     getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        let session = false;
+        let withheld = false;
         const objectStore = txn.objectStore("inbound_group_sessions");
         const getReq = objectStore.get([senderCurve25519Key, sessionId]);
         getReq.onsuccess = function() {
             try {
                 if (getReq.result) {
-                    func(getReq.result.session);
+                    session = getReq.result.session;
                 } else {
-                    func(null);
+                    session = null;
+                }
+                if (withheld !== false) {
+                    func(session, withheld);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+
+        const withheldObjectStore = txn.objectStore("inbound_group_sessions_withheld");
+        const withheldGetReq = withheldObjectStore.get([senderCurve25519Key, sessionId]);
+        withheldGetReq.onsuccess = function() {
+            try {
+                if (withheldGetReq.result) {
+                    withheld = withheldGetReq.result.session;
+                } else {
+                    withheld = null;
+                }
+                if (session !== false) {
+                    func(session, withheld);
                 }
             } catch (e) {
                 abortWithException(txn, e);
@@ -499,6 +522,15 @@ export class Backend {
         });
     }
 
+    storeEndToEndInboundGroupSessionWithheld(
+        senderCurve25519Key, sessionId, sessionData, txn,
+    ) {
+        const objectStore = txn.objectStore("inbound_group_sessions_withheld");
+        objectStore.put({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+    }
+
     getEndToEndDeviceData(txn, func) {
         const objectStore = txn.objectStore("device_data");
         const getReq = objectStore.get("-");
@@ -662,6 +694,11 @@ export function upgradeDatabase(db, oldVersion) {
             keyPath: ["senderCurve25519Key", "sessionId"],
         });
     }
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
     // Expand as needed.
 }
 

src/crypto/store/indexeddb-crypto-store.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -104,7 +105,10 @@ export class IndexedDBCryptoStore {
             // we can fall back to a different backend.
             return backend.doTxn(
                 'readonly',
-                [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                [
+                    IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+                    IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+                ],
                 (txn) => {
                     backend.getEndToEndInboundGroupSession('', '', txn, () => {});
                 }).then(() => {
@@ -471,6 +475,16 @@ export class IndexedDBCryptoStore {
         });
     }
 
+    storeEndToEndInboundGroupSessionWithheld(
+        senderCurve25519Key, sessionId, sessionData, txn,
+    ) {
+        this._backendPromise.then(backend => {
+            backend.storeEndToEndInboundGroupSessionWithheld(
+                senderCurve25519Key, sessionId, sessionData, txn,
+            );
+        });
+    }
+
     // End-to-end device tracking
 
     /**
@@ -607,6 +621,8 @@ export class IndexedDBCryptoStore {
 IndexedDBCryptoStore.STORE_ACCOUNT = 'account';
 IndexedDBCryptoStore.STORE_SESSIONS = 'sessions';
 IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS = 'inbound_group_sessions';
+IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD
+    = 'inbound_group_sessions_withheld';
 IndexedDBCryptoStore.STORE_DEVICE_DATA = 'device_data';
 IndexedDBCryptoStore.STORE_ROOMS = 'rooms';
 IndexedDBCryptoStore.STORE_BACKUP = 'sessions_needing_backup';

src/crypto/store/localStorage-crypto-store.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017, 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -32,6 +33,7 @@ const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
 const KEY_CROSS_SIGNING_KEYS = E2E_PREFIX + "cross_signing_keys";
 const KEY_DEVICE_DATA = E2E_PREFIX + "device_data";
 const KEY_INBOUND_SESSION_PREFIX = E2E_PREFIX + "inboundgroupsessions/";
+const KEY_INBOUND_SESSION_WITHHELD_PREFIX = E2E_PREFIX + "inboundgroupsessions.withheld/";
 const KEY_ROOMS_PREFIX = E2E_PREFIX + "rooms/";
 const KEY_SESSIONS_NEEDING_BACKUP = E2E_PREFIX + "sessionsneedingbackup";
 
@@ -43,6 +45,10 @@ function keyEndToEndInboundGroupSession(senderKey, sessionId) {
     return KEY_INBOUND_SESSION_PREFIX + senderKey + "/" + sessionId;
 }
 
+function keyEndToEndInboundGroupSessionWithheld(senderKey, sessionId) {
+    return KEY_INBOUND_SESSION_WITHHELD_PREFIX + senderKey + "/" + sessionId;
+}
+
 function keyEndToEndRoomsPrefix(roomId) {
     return KEY_ROOMS_PREFIX + roomId;
 }
@@ -125,10 +131,16 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
     // Inbound Group Sessions
 
     getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
-        func(getJsonItem(
+        func(
-            this.store,
+            getJsonItem(
-            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+                this.store,
-        ));
+                keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+            ),
+            getJsonItem(
+                this.store,
+                keyEndToEndInboundGroupSessionWithheld(senderCurve25519Key, sessionId),
+            ),
+        );
     }
 
     getAllEndToEndInboundGroupSessions(txn, func) {
@@ -170,6 +182,16 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
         );
     }
 
+    storeEndToEndInboundGroupSessionWithheld(
+        senderCurve25519Key, sessionId, sessionData, txn,
+    ) {
+        setJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSessionWithheld(senderCurve25519Key, sessionId),
+            sessionData,
+        );
+    }
+
     getEndToEndDeviceData(txn, func) {
         func(getJsonItem(
             this.store, KEY_DEVICE_DATA,

src/crypto/store/memory-crypto-store.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -37,6 +38,7 @@ export class MemoryCryptoStore {
         this._sessions = {};
         // Map of {senderCurve25519Key+'/'+sessionId -> session data object}
         this._inboundGroupSessions = {};
+        this._inboundGroupSessionsWithheld = {};
         // Opaque device data object
         this._deviceData = null;
         // roomId -> Opaque roomInfo object
@@ -276,7 +278,11 @@ export class MemoryCryptoStore {
     // Inbound Group Sessions
 
     getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
-        func(this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] || null);
+        const k = senderCurve25519Key+'/'+sessionId;
+        func(
+            this._inboundGroupSessions[k] || null,
+            this._inboundGroupSessionsWithheld[k] || null,
+        );
     }
 
     getAllEndToEndInboundGroupSessions(txn, func) {
@@ -306,6 +312,13 @@ export class MemoryCryptoStore {
         this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] = sessionData;
     }
 
+    storeEndToEndInboundGroupSessionWithheld(
+        senderCurve25519Key, sessionId, sessionData, txn,
+    ) {
+        const k = senderCurve25519Key+'/'+sessionId;
+        this._inboundGroupSessionsWithheld[k] = sessionData;
+    }
+
     // Device Data
 
     getEndToEndDeviceData(txn, func) {

src/models/room-member.js

@@ -312,10 +312,18 @@ function calculateDisplayName(selfUserId, displayName, roomState) {
     // Next check if the name contains something that look like a mxid
     // If it does, it may be someone trying to impersonate someone else
     // Show full mxid in this case
-    // Also show mxid if there are other people with the same or similar
-    // displayname, after hidden character removal.
     let disambiguate = /@.+:.+/.test(displayName);
+
     if (!disambiguate) {
+        // Also show mxid if the display name contains any LTR/RTL characters as these
+        // make it very difficult for us to find similar *looking* display names
+        // E.g "Mark" could be cloned by writing "kraM" but in RTL.
+        disambiguate = /[\u200E\u200F\u202A-\u202F]/.test(displayName);
+    }
+
+    if (!disambiguate) {
+        // Also show mxid if there are other people with the same or similar
+        // displayname, after hidden character removal.
         const userIds = roomState.getUserIdsWithDisplayName(displayName);
         disambiguate = userIds.some((u) => u !== selfUserId);
     }

src/scheduler.js

@@ -157,6 +157,11 @@ MatrixScheduler.RETRY_BACKOFF_RATELIMIT = function(event, attempts, err) {
         return -1;
     }
 
+    // if event that we are trying to send is too large in any way then retrying won't help
+    if (err.name === "M_TOO_LARGE") {
+        return -1;
+    }
+
     if (err.name === "M_LIMIT_EXCEEDED") {
         const waitTime = err.data.retry_after_ms;
         if (waitTime) {

src/utils.js

@@ -645,8 +645,20 @@ export function isNumber(value) {
  */
 export function removeHiddenChars(str) {
     return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+<<<<<<< HEAD
 }
 const removeHiddenCharsRegex = /[\u200B-\u200D\u0300-\u036f\uFEFF\s]/g;
+=======
+};
+// Regex matching bunch of unicode control characters and otherwise misleading/invisible characters.
+// Includes:
+// various width spaces U+2000 - U+200D
+// LTR and RTL marks U+200E and U+200F
+// LTR/RTL and other directional formatting marks U+202A - U+202F
+// Combining characters U+0300 - U+036F
+// Zero width no-break space (BOM) U+FEFF
+const removeHiddenCharsRegex = /[\u2000-\u200F\u202A-\u202F\u0300-\u036f\uFEFF\s]/g;
+>>>>>>> develop
 
 export function escapeRegExp(string) {
     return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");