Merge pull request #815 from matrix-org/develop

Merge develop into experimental

CONTRIBUTING.rst

@@ -113,3 +113,8 @@ include the line in your commit or pull request comment::
 can't be accepted. Git makes this trivial - just use the -s flag when you do
 ``git commit``, having first set ``user.name`` and ``user.email`` git configs
 (which you should have done anyway :)
+
+If you forgot to sign off your commits before making your pull request and are on git 2.17+ 
+you can mass signoff using rebase::
+
+    git rebase --signoff origin/develop

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.13.1",
+  "version": "0.14.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -6735,6 +6735,11 @@
         }
       }
     },
+    "unhomoglyph": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/unhomoglyph/-/unhomoglyph-1.0.2.tgz",
+      "integrity": "sha1-1p5fWmocayEZQaCIm4HrqGWVwlM="
+    },
     "union-value": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",

package.json

@@ -54,13 +54,15 @@
   "dependencies": {
     "another-json": "^0.2.0",
     "babel-runtime": "^6.26.0",
+    "base-x": "3.0.4",
     "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
     "content-type": "^1.0.2",
     "loglevel": "1.6.1",
     "qs": "^6.5.2",
-    "request": "^2.88.0"
+    "request": "^2.88.0",
+    "unhomoglyph": "^1.0.2"
   },
   "devDependencies": {
     "babel-cli": "^6.18.0",

spec/unit/room-member.spec.js

@@ -285,5 +285,52 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent); // no-op
             expect(emitCount).toEqual(1);
         });
+
+        it("should set 'name' to user_id if it is just whitespace", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: " \u200b ",
+            });
+
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent);
+            expect(member.name).toEqual(userA); // it should fallback because all whitespace
+        });
+
+        it("should disambiguate users on a fuzzy displayname match", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: "Alíce\u200b", // note diacritic and zero width char
+            });
+
+            const roomState = {
+                getStateEvents: function(type) {
+                    if (type !== "m.room.member") {
+                        return [];
+                    }
+                    return [
+                        utils.mkMembership({
+                            event: true, mship: "join", room: roomId,
+                            user: userC, name: "Alice",
+                        }),
+                        joinEvent,
+                    ];
+                },
+                getUserIdsWithDisplayName: function(displayName) {
+                    return [userA, userC];
+                },
+            };
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent, roomState);
+            expect(member.name).toNotEqual("Alíce"); // it should disambig.
+            // user_id should be there somewhere
+            expect(member.name.indexOf(userA)).toNotEqual(-1);
+        });
     });
 });

src/base-apis.js

@@ -1833,7 +1833,7 @@ MatrixBaseApis.prototype.getThirdpartyProtocols = function() {
  * Get information on how a specific place on a third party protocol
  * may be reached.
  * @param {string} protocol The protocol given in getThirdpartyProtocols()
- * @param {object} params Protocol-specific parameters, as given in th
+ * @param {object} params Protocol-specific parameters, as given in the
  *                        response to getThirdpartyProtocols()
  * @return {module:client.Promise} Resolves to the result object
  */
@@ -1848,6 +1848,25 @@ MatrixBaseApis.prototype.getThirdpartyLocation = function(protocol, params) {
     );
 };
 
+/**
+ * Get information on how a specific user on a third party protocol
+ * may be reached.
+ * @param {string} protocol The protocol given in getThirdpartyProtocols()
+ * @param {object} params Protocol-specific parameters, as given in the
+ *                        response to getThirdpartyProtocols()
+ * @return {module:client.Promise} Resolves to the result object
+ */
+MatrixBaseApis.prototype.getThirdpartyUser = function(protocol, params) {
+    const path = utils.encodeUri("/thirdparty/user/$protocol", {
+        $protocol: protocol,
+    });
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "GET", path, params, undefined,
+        httpApi.PREFIX_UNSTABLE,
+    );
+};
+
 /**
  * MatrixBaseApis object
  */

src/client.js

@@ -447,6 +447,7 @@ MatrixClient.prototype.initCrypto = async function() {
     );
 
     this.reEmitter.reEmit(crypto, [
+        "crypto.keyBackupFailed",
         "crypto.roomKeyRequest",
         "crypto.roomKeyRequestCancellation",
         "crypto.warning",
@@ -2262,6 +2263,27 @@ MatrixClient.prototype.mxcUrlToHttp =
     );
 };
 
+/**
+ * Sets a new status message for the user. The message may be null/falsey
+ * to clear the message.
+ * @param {string} newMessage The new message to set.
+ * @return {module:client.Promise} Resolves: to nothing
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype._unstable_setStatusMessage = function(newMessage) {
+    return Promise.all(this.getRooms().map((room) => {
+        const isJoined = room.getMyMembership() === "join";
+        const looksLikeDm = room.getInvitedAndJoinedMemberCount() === 2;
+        if (isJoined && looksLikeDm) {
+            return this.sendStateEvent(room.roomId, "im.vector.user_status", {
+                status: newMessage,
+            }, this.getUserId());
+        } else {
+            return Promise.resolve();
+        }
+    }));
+};
+
 /**
  * @param {Object} opts Options to apply
  * @param {string} opts.presence One of "online", "offline" or "unavailable"

src/crypto/index.js

@@ -42,6 +42,7 @@ export function isCryptoAvailable() {
 }
 
 const MIN_FORCE_SESSION_INTERVAL_MS = 60 * 60 * 1000;
+const KEY_BACKUP_KEYS_PER_REQUEST = 200;
 
 /**
  * Cryptography bits
@@ -986,29 +987,70 @@ Crypto.prototype.importRoomKeys = function(keys) {
     );
 };
 
-Crypto.prototype._maybeSendKeyBackup = async function(delay, retry) {
+/**
-    if (retry === undefined) retry = true;
+ * Schedules sending all keys waiting to be sent to the backup, if not already
+ * scheduled. Retries if necessary.
+ */
+Crypto.prototype._scheduleKeyBackupSend = async function() {
+    if (this._sendingBackups) return;
 
-    if (!this._sendingBackups) {
-        this._sendingBackups = true;
     try {
-            if (delay === undefined) {
+        // wait between 0 and 10 seconds, to avoid backup
-                // by default, wait between 0 and 10 seconds, to avoid backup
         // requests from different clients hitting the server all at
         // the same time when a new key is sent
-                delay = Math.random() * 10000;
+        const delay = Math.random() * 10000;
-            }
         await Promise.delay(delay);
         let numFailures = 0; // number of consecutive failures
         while (1) {
             if (!this.backupKey) {
                 return;
             }
-                // FIXME: figure out what limit is reasonable
+            try {
-                const sessions = await this._cryptoStore.getSessionsNeedingBackup(10);
+                const numBackedUp =
-                if (!sessions.length) {
+                    await this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST);
+                if (numBackedUp === 0) {
+                    // no sessions left needing backup: we're done
                     return;
                 }
+                numFailures = 0;
+            } catch (err) {
+                numFailures++;
+                console.log("Key backup request failed", err);
+                if (err.data) {
+                    if (
+                        err.data.errcode == 'M_NOT_FOUND' ||
+                        err.data.errcode == 'M_WRONG_ROOM_KEYS_VERSION'
+                    ) {
+                        // Backup version has changed or this backup version
+                        // has been deleted
+                        this.emit("crypto.keyBackupFailed", err.data.errcode);
+                        throw err;
+                    }
+                }
+            }
+            if (numFailures) {
+                // exponential backoff if we have failures
+                await Promise.delay(1000 * Math.pow(2, Math.min(numFailures - 1, 4)));
+            }
+        }
+    } finally {
+        this._sendingBackups = false;
+    }
+};
+
+/**
+ * Take some e2e keys waiting to be backed up and send them
+ * to the backup.
+ *
+ * @param {integer} limit Maximum number of keys to back up
+ * @returns {integer} Number of sessions backed up
+ */
+Crypto.prototype._backupPendingKeys = async function(limit) {
+    const sessions = await this._cryptoStore.getSessionsNeedingBackup(limit);
+    if (!sessions.length) {
+        return 0;
+    }
+
     const data = {};
     for (const session of sessions) {
         const roomId = session.sessionData.room_id;
@@ -1041,39 +1083,13 @@ Crypto.prototype._maybeSendKeyBackup = async function(delay, retry) {
         };
     }
 
-                try {
     await this._baseApis.sendKeyBackup(
         undefined, undefined, this.backupInfo.version,
         {rooms: data},
     );
-                    numFailures = 0;
     await this._cryptoStore.unmarkSessionsNeedingBackup(sessions);
-                } catch (err) {
+
-                    numFailures++;
+    return sessions.length;
-                    console.log("send failed", err);
-                    if (err.httpStatus === 400
-                        || err.httpStatus === 403
-                        || err.httpStatus === 401
-                        || !retry) {
-                        // retrying probably won't help much, so we should give up
-                        // FIXME: disable backups?
-                        throw err;
-                    }
-                }
-                if (numFailures) {
-                    // exponential backoff if we have failures
-                    await new Promise((resolve, reject) => {
-                        setTimeout(
-                            resolve,
-                            1000 * Math.pow(2, Math.min(numFailures - 1, 4)),
-                        );
-                    });
-                }
-            }
-        } finally {
-            this._sendingBackups = false;
-        }
-    }
 };
 
 Crypto.prototype.backupGroupSession = async function(
@@ -1090,7 +1106,9 @@ Crypto.prototype.backupGroupSession = async function(
         sessionId: sessionId,
     }]);
 
-    await this._maybeSendKeyBackup();
+    // don't wait for this to complete: it will delay so
+    // happens in the background
+    this._scheduleKeyBackupSend();
 };
 
 Crypto.prototype.backupAllGroupSessions = async function(version) {
@@ -1109,7 +1127,10 @@ Crypto.prototype.backupAllGroupSessions = async function(version) {
         },
     );
 
-    await this._maybeSendKeyBackup(0, false);
+    let numKeysBackedUp;
+    do {
+        numKeysBackedUp = await this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST);
+    } while (numKeysBackedUp > 0);
 };
 
 /* eslint-disable valid-jsdoc */    //https://github.com/eslint/eslint/issues/7307

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -384,6 +384,24 @@ export class Backend {
         };
     }
 
+    getAllEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                func(cursor.value);
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
     storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
         const objectStore = txn.objectStore("sessions");
         objectStore.put({

src/crypto/store/indexeddb-crypto-store.js

@@ -336,6 +336,17 @@ export default class IndexedDBCryptoStore {
         this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func);
     }
 
+    /**
+     * Retrieve all end-to-end sessions
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called one for each session with
+     *     an object with, deviceKey, lastReceivedMessageTs, sessionId
+     *     and session keys.
+     */
+    getAllEndToEndSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndSessions(txn, func);
+    }
+
     /**
      * Store a session between the logged-in user and another device
      * @param {string} deviceKey The public key of the other device.

src/crypto/store/localStorage-crypto-store.js

@@ -94,6 +94,17 @@ export default class LocalStorageCryptoStore extends MemoryCryptoStore {
         func(this._getEndToEndSessions(deviceKey) || {});
     }
 
+    getAllEndToEndSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) {
+                const deviceKey = this.store.key(i).split('/')[1];
+                for (const sess of Object.values(this._getEndToEndSessions(deviceKey))) {
+                    func(sess);
+                }
+            }
+        }
+    }
+
     storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
         const sessions = this._getEndToEndSessions(deviceKey) || {};
         sessions[sessionId] = sessionInfo;

src/crypto/store/memory-crypto-store.js

@@ -249,6 +249,14 @@ export default class MemoryCryptoStore {
         func(this._sessions[deviceKey] || {});
     }
 
+    getAllEndToEndSessions(txn, func) {
+        for (const deviceSessions of Object.values(this._sessions)) {
+            for (const sess of Object.values(deviceSessions)) {
+                func(sess);
+            }
+        }
+    }
+
     storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
         let deviceSessions = this._sessions[deviceKey];
         if (deviceSessions === undefined) {

src/models/room-member.js

@@ -298,26 +298,24 @@ function calculateDisplayName(selfUserId, displayName, roomState) {
         return selfUserId;
     }
 
-    if (!roomState) {
-        return displayName;
-    }
-
     // First check if the displayname is something we consider truthy
     // after stripping it of zero width characters and padding spaces
-    const strippedDisplayName = utils.removeHiddenChars(displayName);
+    if (!utils.removeHiddenChars(displayName)) {
-    if (!strippedDisplayName) {
         return selfUserId;
     }
 
+    if (!roomState) {
+        return displayName;
+    }
+
     // Next check if the name contains something that look like a mxid
     // If it does, it may be someone trying to impersonate someone else
     // Show full mxid in this case
-    // Also show mxid if there are other people with the same displayname
+    // Also show mxid if there are other people with the same or similar
-    // ignoring any zero width chars (unicode 200B-200D)
+    // displayname, after hidden character removal.
-    // if their displayname is made up of just zero width chars, show full mxid
     let disambiguate = /@.+:.+/.test(displayName);
     if (!disambiguate) {
-        const userIds = roomState.getUserIdsWithDisplayName(strippedDisplayName);
+        const userIds = roomState.getUserIdsWithDisplayName(displayName);
         disambiguate = userIds.some((u) => u !== selfUserId);
     }
 

src/models/room-state.js

@@ -75,6 +75,8 @@ function RoomState(roomId, oobMemberFlags = undefined) {
         // userId: RoomMember
     };
     this._updateModifiedTime();
+
+    // stores fuzzy matches to a list of userIDs (applies utils.removeHiddenChars to keys)
     this._displayNameToUserIds = {};
     this._userIdsToDisplayNames = {};
     this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite
@@ -154,6 +156,16 @@ RoomState.prototype.getMembers = function() {
     return utils.values(this.members);
 };
 
+/**
+ * Get all RoomMembers in this room, excluding the user IDs provided.
+ * @param {Array<string>} excludedIds The user IDs to exclude.
+ * @return {Array<RoomMember>} A list of RoomMembers.
+ */
+RoomState.prototype.getMembersExcept = function(excludedIds) {
+    return utils.values(this.members)
+        .filter((m) => !excludedIds.includes(m.userId));
+};
+
 /**
  * Get a room member by their user ID.
  * @param {string} userId The room member's user ID.
@@ -519,12 +531,12 @@ RoomState.prototype.getLastModifiedTime = function() {
 };
 
 /**
- * Get user IDs with the specified display name.
+ * Get user IDs with the specified or similar display names.
  * @param {string} displayName The display name to get user IDs from.
  * @return {string[]} An array of user IDs or an empty array.
  */
 RoomState.prototype.getUserIdsWithDisplayName = function(displayName) {
-    return this._displayNameToUserIds[displayName] || [];
+    return this._displayNameToUserIds[utils.removeHiddenChars(displayName)] || [];
 };
 
 /**

src/models/room.js

@@ -1167,7 +1167,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
         this.removeEvent(oldEventId);
     }
 
-    this.emit("Room.localEchoUpdated", event, this, event.getId(), oldStatus);
+    this.emit("Room.localEchoUpdated", event, this, oldEventId, oldStatus);
 };
 
 

src/models/user.js

@@ -39,6 +39,9 @@ limitations under the License.
  *                when a user was last active.
  * @prop {Boolean} currentlyActive Whether we should consider lastActiveAgo to be
  *               an approximation and that the user should be seen as active 'now'
+ * @prop {string} _unstable_statusMessage The status message for the user, if known. This is
+ *                different from the presenceStatusMsg in that this is not tied to
+ *                the user's presence, and should be represented differently.
  * @prop {Object} events The events describing this user.
  * @prop {MatrixEvent} events.presence The m.presence event for this user.
  */
@@ -46,6 +49,7 @@ function User(userId) {
     this.userId = userId;
     this.presence = "offline";
     this.presenceStatusMsg = null;
+    this._unstable_statusMessage = "";
     this.displayName = userId;
     this.rawDisplayName = userId;
     this.avatarUrl = null;
@@ -179,6 +183,16 @@ User.prototype.getLastActiveTs = function() {
     return this.lastPresenceTs - this.lastActiveAgo;
 };
 
+/**
+ * Manually set the user's status message.
+ * @param {MatrixEvent} event The <code>im.vector.user_status</code> event.
+ */
+User.prototype._unstable_updateStatusMessage = function(event) {
+    if (!event.getContent()) this._unstable_statusMessage = "";
+    else this._unstable_statusMessage = event.getContent()["status"];
+    this._updateModifiedTime();
+};
+
 /**
  * The User class.
  */

src/sync.js

@@ -1172,6 +1172,16 @@ SyncApi.prototype._processSyncResponse = async function(
             if (e.isState() && e.getType() == "m.room.encryption" && self.opts.crypto) {
                 await self.opts.crypto.onCryptoEvent(e);
             }
+            if (e.isState() && e.getType() === "im.vector.user_status") {
+                let user = client.store.getUser(e.getStateKey());
+                if (user) {
+                    user._unstable_updateStatusMessage(e);
+                } else {
+                    user = createNewUser(client, e.getStateKey());
+                    user._unstable_updateStatusMessage(e);
+                    client.store.storeUser(user);
+                }
+            }
         }
 
         await Promise.mapSeries(stateEvents, processRoomEvent);

src/utils.js

@@ -19,6 +19,8 @@ limitations under the License.
  * @module utils
  */
 
+const unhomoglyph = require('unhomoglyph');
+
 /**
  * Encode a dictionary of query parameters.
  * @param {Object} params A dict of key/values to encode e.g.
@@ -665,11 +667,12 @@ module.exports.isNumber = function(value) {
 
 /**
  * Removes zero width chars, diacritics and whitespace from the string
+ * Also applies an unhomoglyph on the string, to prevent similar looking chars
  * @param {string} str the string to remove hidden characters from
  * @return {string} a string with the hidden characters removed
  */
 module.exports.removeHiddenChars = function(str) {
-    return str.normalize('NFD').replace(removeHiddenCharsRegex, '');
+    return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
 };
 const removeHiddenCharsRegex = /[\u200B-\u200D\u0300-\u036f\uFEFF\s]/g;