diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 3100fb1d1..436136430 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -113,3 +113,8 @@ include the line in your commit or pull request comment:: can't be accepted. Git makes this trivial - just use the -s flag when you do ``git commit``, having first set ``user.name`` and ``user.email`` git configs (which you should have done anyway :) + +If you forgot to sign off your commits before making your pull request and are on git 2.17+ +you can mass signoff using rebase:: + + git rebase --signoff origin/develop diff --git a/package-lock.json b/package-lock.json index df95672aa..e950a7ff1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "0.13.1", + "version": "0.14.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -6735,6 +6735,11 @@ } } }, + "unhomoglyph": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unhomoglyph/-/unhomoglyph-1.0.2.tgz", + "integrity": "sha1-1p5fWmocayEZQaCIm4HrqGWVwlM=" + }, "union-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", diff --git a/package.json b/package.json index 03fdc40ff..71a43fc99 100644 --- a/package.json +++ b/package.json @@ -54,13 +54,15 @@ "dependencies": { "another-json": "^0.2.0", "babel-runtime": "^6.26.0", + "base-x": "3.0.4", "bluebird": "^3.5.0", "browser-request": "^0.3.3", "bs58": "^4.0.1", "content-type": "^1.0.2", "loglevel": "1.6.1", "qs": "^6.5.2", - "request": "^2.88.0" + "request": "^2.88.0", + "unhomoglyph": "^1.0.2" }, "devDependencies": { "babel-cli": "^6.18.0", diff --git a/spec/unit/room-member.spec.js b/spec/unit/room-member.spec.js index 298771128..77c2f7058 100644 --- a/spec/unit/room-member.spec.js +++ b/spec/unit/room-member.spec.js @@ -285,5 +285,52 @@ describe("RoomMember", function() { member.setMembershipEvent(joinEvent); // no-op expect(emitCount).toEqual(1); }); + + it("should set 'name' to user_id if it is just whitespace", function() { + const joinEvent = utils.mkMembership({ + event: true, + mship: "join", + user: userA, + room: roomId, + name: " \u200b ", + }); + + expect(member.name).toEqual(userA); // default = user_id + member.setMembershipEvent(joinEvent); + expect(member.name).toEqual(userA); // it should fallback because all whitespace + }); + + it("should disambiguate users on a fuzzy displayname match", function() { + const joinEvent = utils.mkMembership({ + event: true, + mship: "join", + user: userA, + room: roomId, + name: "Alíce\u200b", // note diacritic and zero width char + }); + + const roomState = { + getStateEvents: function(type) { + if (type !== "m.room.member") { + return []; + } + return [ + utils.mkMembership({ + event: true, mship: "join", room: roomId, + user: userC, name: "Alice", + }), + joinEvent, + ]; + }, + getUserIdsWithDisplayName: function(displayName) { + return [userA, userC]; + }, + }; + expect(member.name).toEqual(userA); // default = user_id + member.setMembershipEvent(joinEvent, roomState); + expect(member.name).toNotEqual("Alíce"); // it should disambig. + // user_id should be there somewhere + expect(member.name.indexOf(userA)).toNotEqual(-1); + }); }); }); diff --git a/src/base-apis.js b/src/base-apis.js index 8fd1bd0f1..4e9fed77b 100644 --- a/src/base-apis.js +++ b/src/base-apis.js @@ -1833,7 +1833,7 @@ MatrixBaseApis.prototype.getThirdpartyProtocols = function() { * Get information on how a specific place on a third party protocol * may be reached. * @param {string} protocol The protocol given in getThirdpartyProtocols() - * @param {object} params Protocol-specific parameters, as given in th + * @param {object} params Protocol-specific parameters, as given in the * response to getThirdpartyProtocols() * @return {module:client.Promise} Resolves to the result object */ @@ -1848,6 +1848,25 @@ MatrixBaseApis.prototype.getThirdpartyLocation = function(protocol, params) { ); }; +/** + * Get information on how a specific user on a third party protocol + * may be reached. + * @param {string} protocol The protocol given in getThirdpartyProtocols() + * @param {object} params Protocol-specific parameters, as given in the + * response to getThirdpartyProtocols() + * @return {module:client.Promise} Resolves to the result object + */ +MatrixBaseApis.prototype.getThirdpartyUser = function(protocol, params) { + const path = utils.encodeUri("/thirdparty/user/$protocol", { + $protocol: protocol, + }); + + return this._http.authedRequestWithPrefix( + undefined, "GET", path, params, undefined, + httpApi.PREFIX_UNSTABLE, + ); +}; + /** * MatrixBaseApis object */ diff --git a/src/client.js b/src/client.js index 22cae3e87..339333319 100644 --- a/src/client.js +++ b/src/client.js @@ -447,6 +447,7 @@ MatrixClient.prototype.initCrypto = async function() { ); this.reEmitter.reEmit(crypto, [ + "crypto.keyBackupFailed", "crypto.roomKeyRequest", "crypto.roomKeyRequestCancellation", "crypto.warning", @@ -2262,6 +2263,27 @@ MatrixClient.prototype.mxcUrlToHttp = ); }; +/** + * Sets a new status message for the user. The message may be null/falsey + * to clear the message. + * @param {string} newMessage The new message to set. + * @return {module:client.Promise} Resolves: to nothing + * @return {module:http-api.MatrixError} Rejects: with an error response. + */ +MatrixClient.prototype._unstable_setStatusMessage = function(newMessage) { + return Promise.all(this.getRooms().map((room) => { + const isJoined = room.getMyMembership() === "join"; + const looksLikeDm = room.getInvitedAndJoinedMemberCount() === 2; + if (isJoined && looksLikeDm) { + return this.sendStateEvent(room.roomId, "im.vector.user_status", { + status: newMessage, + }, this.getUserId()); + } else { + return Promise.resolve(); + } + })); +}; + /** * @param {Object} opts Options to apply * @param {string} opts.presence One of "online", "offline" or "unavailable" diff --git a/src/crypto/index.js b/src/crypto/index.js index 17abf8933..9e8daf93e 100644 --- a/src/crypto/index.js +++ b/src/crypto/index.js @@ -42,6 +42,7 @@ export function isCryptoAvailable() { } const MIN_FORCE_SESSION_INTERVAL_MS = 60 * 60 * 1000; +const KEY_BACKUP_KEYS_PER_REQUEST = 200; /** * Cryptography bits @@ -986,96 +987,111 @@ Crypto.prototype.importRoomKeys = function(keys) { ); }; -Crypto.prototype._maybeSendKeyBackup = async function(delay, retry) { - if (retry === undefined) retry = true; +/** + * Schedules sending all keys waiting to be sent to the backup, if not already + * scheduled. Retries if necessary. + */ +Crypto.prototype._scheduleKeyBackupSend = async function() { + if (this._sendingBackups) return; - if (!this._sendingBackups) { - this._sendingBackups = true; - try { - if (delay === undefined) { - // by default, wait between 0 and 10 seconds, to avoid backup - // requests from different clients hitting the server all at - // the same time when a new key is sent - delay = Math.random() * 10000; + try { + // wait between 0 and 10 seconds, to avoid backup + // requests from different clients hitting the server all at + // the same time when a new key is sent + const delay = Math.random() * 10000; + await Promise.delay(delay); + let numFailures = 0; // number of consecutive failures + while (1) { + if (!this.backupKey) { + return; } - await Promise.delay(delay); - let numFailures = 0; // number of consecutive failures - while (1) { - if (!this.backupKey) { + try { + const numBackedUp = + await this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST); + if (numBackedUp === 0) { + // no sessions left needing backup: we're done return; } - // FIXME: figure out what limit is reasonable - const sessions = await this._cryptoStore.getSessionsNeedingBackup(10); - if (!sessions.length) { - return; - } - const data = {}; - for (const session of sessions) { - const roomId = session.sessionData.room_id; - if (data[roomId] === undefined) { - data[roomId] = {sessions: {}}; - } - - const sessionData = await this._olmDevice.exportInboundGroupSession( - session.senderKey, session.sessionId, session.sessionData, - ); - sessionData.algorithm = olmlib.MEGOLM_ALGORITHM; - delete sessionData.session_id; - delete sessionData.room_id; - const firstKnownIndex = sessionData.first_known_index; - delete sessionData.first_known_index; - const encrypted = this.backupKey.encrypt(JSON.stringify(sessionData)); - - const forwardedCount = - (sessionData.forwarding_curve25519_key_chain || []).length; - - const device = this._deviceList.getDeviceByIdentityKey( - olmlib.MEGOLM_ALGORITHM, session.senderKey, - ); - - data[roomId]['sessions'][session.sessionId] = { - first_message_index: firstKnownIndex, - forwarded_count: forwardedCount, - is_verified: !!(device && device.isVerified()), - session_data: encrypted, - }; - } - - try { - await this._baseApis.sendKeyBackup( - undefined, undefined, this.backupInfo.version, - {rooms: data}, - ); - numFailures = 0; - await this._cryptoStore.unmarkSessionsNeedingBackup(sessions); - } catch (err) { - numFailures++; - console.log("send failed", err); - if (err.httpStatus === 400 - || err.httpStatus === 403 - || err.httpStatus === 401 - || !retry) { - // retrying probably won't help much, so we should give up - // FIXME: disable backups? + numFailures = 0; + } catch (err) { + numFailures++; + console.log("Key backup request failed", err); + if (err.data) { + if ( + err.data.errcode == 'M_NOT_FOUND' || + err.data.errcode == 'M_WRONG_ROOM_KEYS_VERSION' + ) { + // Backup version has changed or this backup version + // has been deleted + this.emit("crypto.keyBackupFailed", err.data.errcode); throw err; } } - if (numFailures) { - // exponential backoff if we have failures - await new Promise((resolve, reject) => { - setTimeout( - resolve, - 1000 * Math.pow(2, Math.min(numFailures - 1, 4)), - ); - }); - } } - } finally { - this._sendingBackups = false; + if (numFailures) { + // exponential backoff if we have failures + await Promise.delay(1000 * Math.pow(2, Math.min(numFailures - 1, 4))); + } } + } finally { + this._sendingBackups = false; } }; +/** + * Take some e2e keys waiting to be backed up and send them + * to the backup. + * + * @param {integer} limit Maximum number of keys to back up + * @returns {integer} Number of sessions backed up + */ +Crypto.prototype._backupPendingKeys = async function(limit) { + const sessions = await this._cryptoStore.getSessionsNeedingBackup(limit); + if (!sessions.length) { + return 0; + } + + const data = {}; + for (const session of sessions) { + const roomId = session.sessionData.room_id; + if (data[roomId] === undefined) { + data[roomId] = {sessions: {}}; + } + + const sessionData = await this._olmDevice.exportInboundGroupSession( + session.senderKey, session.sessionId, session.sessionData, + ); + sessionData.algorithm = olmlib.MEGOLM_ALGORITHM; + delete sessionData.session_id; + delete sessionData.room_id; + const firstKnownIndex = sessionData.first_known_index; + delete sessionData.first_known_index; + const encrypted = this.backupKey.encrypt(JSON.stringify(sessionData)); + + const forwardedCount = + (sessionData.forwarding_curve25519_key_chain || []).length; + + const device = this._deviceList.getDeviceByIdentityKey( + olmlib.MEGOLM_ALGORITHM, session.senderKey, + ); + + data[roomId]['sessions'][session.sessionId] = { + first_message_index: firstKnownIndex, + forwarded_count: forwardedCount, + is_verified: !!(device && device.isVerified()), + session_data: encrypted, + }; + } + + await this._baseApis.sendKeyBackup( + undefined, undefined, this.backupInfo.version, + {rooms: data}, + ); + await this._cryptoStore.unmarkSessionsNeedingBackup(sessions); + + return sessions.length; +}; + Crypto.prototype.backupGroupSession = async function( roomId, senderKey, forwardingCurve25519KeyChain, sessionId, sessionKey, keysClaimed, @@ -1090,7 +1106,9 @@ Crypto.prototype.backupGroupSession = async function( sessionId: sessionId, }]); - await this._maybeSendKeyBackup(); + // don't wait for this to complete: it will delay so + // happens in the background + this._scheduleKeyBackupSend(); }; Crypto.prototype.backupAllGroupSessions = async function(version) { @@ -1109,7 +1127,10 @@ Crypto.prototype.backupAllGroupSessions = async function(version) { }, ); - await this._maybeSendKeyBackup(0, false); + let numKeysBackedUp; + do { + numKeysBackedUp = await this._backupPendingKeys(KEY_BACKUP_KEYS_PER_REQUEST); + } while (numKeysBackedUp > 0); }; /* eslint-disable valid-jsdoc */ //https://github.com/eslint/eslint/issues/7307 diff --git a/src/crypto/store/indexeddb-crypto-store-backend.js b/src/crypto/store/indexeddb-crypto-store-backend.js index 566ef9384..5378efd78 100644 --- a/src/crypto/store/indexeddb-crypto-store-backend.js +++ b/src/crypto/store/indexeddb-crypto-store-backend.js @@ -384,6 +384,24 @@ export class Backend { }; } + getAllEndToEndSessions(txn, func) { + const objectStore = txn.objectStore("sessions"); + const getReq = objectStore.openCursor(); + getReq.onsuccess = function() { + const cursor = getReq.result; + if (cursor) { + func(cursor.value); + cursor.continue(); + } else { + try { + func(null); + } catch (e) { + abortWithException(txn, e); + } + } + }; + } + storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) { const objectStore = txn.objectStore("sessions"); objectStore.put({ diff --git a/src/crypto/store/indexeddb-crypto-store.js b/src/crypto/store/indexeddb-crypto-store.js index bf380b6da..fecc16789 100644 --- a/src/crypto/store/indexeddb-crypto-store.js +++ b/src/crypto/store/indexeddb-crypto-store.js @@ -336,6 +336,17 @@ export default class IndexedDBCryptoStore { this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func); } + /** + * Retrieve all end-to-end sessions + * @param {*} txn An active transaction. See doTxn(). + * @param {function(object)} func Called one for each session with + * an object with, deviceKey, lastReceivedMessageTs, sessionId + * and session keys. + */ + getAllEndToEndSessions(txn, func) { + this._backendPromise.value().getAllEndToEndSessions(txn, func); + } + /** * Store a session between the logged-in user and another device * @param {string} deviceKey The public key of the other device. diff --git a/src/crypto/store/localStorage-crypto-store.js b/src/crypto/store/localStorage-crypto-store.js index 5ca48a16c..45c4baff0 100644 --- a/src/crypto/store/localStorage-crypto-store.js +++ b/src/crypto/store/localStorage-crypto-store.js @@ -94,6 +94,17 @@ export default class LocalStorageCryptoStore extends MemoryCryptoStore { func(this._getEndToEndSessions(deviceKey) || {}); } + getAllEndToEndSessions(txn, func) { + for (let i = 0; i < this.store.length; ++i) { + if (this.store.key(i).startsWith(keyEndToEndSessions(''))) { + const deviceKey = this.store.key(i).split('/')[1]; + for (const sess of Object.values(this._getEndToEndSessions(deviceKey))) { + func(sess); + } + } + } + } + storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) { const sessions = this._getEndToEndSessions(deviceKey) || {}; sessions[sessionId] = sessionInfo; diff --git a/src/crypto/store/memory-crypto-store.js b/src/crypto/store/memory-crypto-store.js index 03753e0e1..0b6e0b6fe 100644 --- a/src/crypto/store/memory-crypto-store.js +++ b/src/crypto/store/memory-crypto-store.js @@ -249,6 +249,14 @@ export default class MemoryCryptoStore { func(this._sessions[deviceKey] || {}); } + getAllEndToEndSessions(txn, func) { + for (const deviceSessions of Object.values(this._sessions)) { + for (const sess of Object.values(deviceSessions)) { + func(sess); + } + } + } + storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) { let deviceSessions = this._sessions[deviceKey]; if (deviceSessions === undefined) { diff --git a/src/models/room-member.js b/src/models/room-member.js index e7a4bf88c..bbc7b5e23 100644 --- a/src/models/room-member.js +++ b/src/models/room-member.js @@ -298,26 +298,24 @@ function calculateDisplayName(selfUserId, displayName, roomState) { return selfUserId; } - if (!roomState) { - return displayName; - } - // First check if the displayname is something we consider truthy // after stripping it of zero width characters and padding spaces - const strippedDisplayName = utils.removeHiddenChars(displayName); - if (!strippedDisplayName) { + if (!utils.removeHiddenChars(displayName)) { return selfUserId; } + if (!roomState) { + return displayName; + } + // Next check if the name contains something that look like a mxid // If it does, it may be someone trying to impersonate someone else // Show full mxid in this case - // Also show mxid if there are other people with the same displayname - // ignoring any zero width chars (unicode 200B-200D) - // if their displayname is made up of just zero width chars, show full mxid + // Also show mxid if there are other people with the same or similar + // displayname, after hidden character removal. let disambiguate = /@.+:.+/.test(displayName); if (!disambiguate) { - const userIds = roomState.getUserIdsWithDisplayName(strippedDisplayName); + const userIds = roomState.getUserIdsWithDisplayName(displayName); disambiguate = userIds.some((u) => u !== selfUserId); } diff --git a/src/models/room-state.js b/src/models/room-state.js index ac5e20077..95da5396b 100644 --- a/src/models/room-state.js +++ b/src/models/room-state.js @@ -75,6 +75,8 @@ function RoomState(roomId, oobMemberFlags = undefined) { // userId: RoomMember }; this._updateModifiedTime(); + + // stores fuzzy matches to a list of userIDs (applies utils.removeHiddenChars to keys) this._displayNameToUserIds = {}; this._userIdsToDisplayNames = {}; this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite @@ -154,6 +156,16 @@ RoomState.prototype.getMembers = function() { return utils.values(this.members); }; +/** + * Get all RoomMembers in this room, excluding the user IDs provided. + * @param {Array} excludedIds The user IDs to exclude. + * @return {Array} A list of RoomMembers. + */ +RoomState.prototype.getMembersExcept = function(excludedIds) { + return utils.values(this.members) + .filter((m) => !excludedIds.includes(m.userId)); +}; + /** * Get a room member by their user ID. * @param {string} userId The room member's user ID. @@ -519,12 +531,12 @@ RoomState.prototype.getLastModifiedTime = function() { }; /** - * Get user IDs with the specified display name. + * Get user IDs with the specified or similar display names. * @param {string} displayName The display name to get user IDs from. * @return {string[]} An array of user IDs or an empty array. */ RoomState.prototype.getUserIdsWithDisplayName = function(displayName) { - return this._displayNameToUserIds[displayName] || []; + return this._displayNameToUserIds[utils.removeHiddenChars(displayName)] || []; }; /** diff --git a/src/models/room.js b/src/models/room.js index 1235a6859..0a835ec64 100644 --- a/src/models/room.js +++ b/src/models/room.js @@ -1167,7 +1167,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) { this.removeEvent(oldEventId); } - this.emit("Room.localEchoUpdated", event, this, event.getId(), oldStatus); + this.emit("Room.localEchoUpdated", event, this, oldEventId, oldStatus); }; diff --git a/src/models/user.js b/src/models/user.js index dbfaeb791..ec2f495af 100644 --- a/src/models/user.js +++ b/src/models/user.js @@ -39,6 +39,9 @@ limitations under the License. * when a user was last active. * @prop {Boolean} currentlyActive Whether we should consider lastActiveAgo to be * an approximation and that the user should be seen as active 'now' + * @prop {string} _unstable_statusMessage The status message for the user, if known. This is + * different from the presenceStatusMsg in that this is not tied to + * the user's presence, and should be represented differently. * @prop {Object} events The events describing this user. * @prop {MatrixEvent} events.presence The m.presence event for this user. */ @@ -46,6 +49,7 @@ function User(userId) { this.userId = userId; this.presence = "offline"; this.presenceStatusMsg = null; + this._unstable_statusMessage = ""; this.displayName = userId; this.rawDisplayName = userId; this.avatarUrl = null; @@ -179,6 +183,16 @@ User.prototype.getLastActiveTs = function() { return this.lastPresenceTs - this.lastActiveAgo; }; +/** + * Manually set the user's status message. + * @param {MatrixEvent} event The im.vector.user_status event. + */ +User.prototype._unstable_updateStatusMessage = function(event) { + if (!event.getContent()) this._unstable_statusMessage = ""; + else this._unstable_statusMessage = event.getContent()["status"]; + this._updateModifiedTime(); +}; + /** * The User class. */ diff --git a/src/sync.js b/src/sync.js index 1ce75ba82..f4ea0bf88 100644 --- a/src/sync.js +++ b/src/sync.js @@ -1172,6 +1172,16 @@ SyncApi.prototype._processSyncResponse = async function( if (e.isState() && e.getType() == "m.room.encryption" && self.opts.crypto) { await self.opts.crypto.onCryptoEvent(e); } + if (e.isState() && e.getType() === "im.vector.user_status") { + let user = client.store.getUser(e.getStateKey()); + if (user) { + user._unstable_updateStatusMessage(e); + } else { + user = createNewUser(client, e.getStateKey()); + user._unstable_updateStatusMessage(e); + client.store.storeUser(user); + } + } } await Promise.mapSeries(stateEvents, processRoomEvent); diff --git a/src/utils.js b/src/utils.js index 8ee81ed3e..265ce218e 100644 --- a/src/utils.js +++ b/src/utils.js @@ -19,6 +19,8 @@ limitations under the License. * @module utils */ +const unhomoglyph = require('unhomoglyph'); + /** * Encode a dictionary of query parameters. * @param {Object} params A dict of key/values to encode e.g. @@ -665,11 +667,12 @@ module.exports.isNumber = function(value) { /** * Removes zero width chars, diacritics and whitespace from the string + * Also applies an unhomoglyph on the string, to prevent similar looking chars * @param {string} str the string to remove hidden characters from * @return {string} a string with the hidden characters removed */ module.exports.removeHiddenChars = function(str) { - return str.normalize('NFD').replace(removeHiddenCharsRegex, ''); + return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, '')); }; const removeHiddenCharsRegex = /[\u200B-\u200D\u0300-\u036f\uFEFF\s]/g;