Merge branch 'develop' into dbkr/cross_signing

.babelrc

@@ -2,6 +2,7 @@
     "presets": ["es2015", "es2016"],
     "plugins": [
         "transform-class-properties",
+
         // this transforms async functions into generator functions, which
         // are then made to use the regenerator module by babel's
         // transform-regnerator plugin (which is enabled by es2015).

.buildkite/pipeline.yaml

@@ -22,3 +22,13 @@ steps:
     plugins:
       - docker#v3.0.1:
           image: "node:10"
+
+  - wait
+
+  - label: "🐴 Trigger matrix-react-sdk"
+    trigger: "matrix-react-sdk"
+    branches: "develop"
+    build:
+        branch: "develop"
+        message: "[js-sdk] ${BUILDKITE_MESSAGE}"
+    async: true

.eslintrc.js

@@ -14,6 +14,9 @@ module.exports = {
         es6: true,
     },
     extends: ["eslint:recommended", "google"],
+    plugins: [
+        "babel",
+    ],
     rules: {
         // rules we've always adhered to or now do
         "max-len": ["error", {
@@ -50,6 +53,7 @@ module.exports = {
         // rules we do not want from the google styleguide
         "object-curly-spacing": ["off"],
         "spaced-comment": ["off"],
+        "guard-for-in": ["off"],
 
         // in principle we prefer single quotes, but life is too short
         quotes: ["off"],
@@ -73,5 +77,10 @@ module.exports = {
             "asyncArrow": "always",
         }],
         "arrow-parens": "off",
+
+        // eslint's built in no-invalid-this rule breaks with class properties
+        "no-invalid-this": "off",
+        // so we replace it with a version that is class property aware
+        "babel/no-invalid-this": "error",
     }
 }

CHANGELOG.md

@@ -1,3 +1,128 @@
+Changes in [2.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.0) (2019-05-31)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.2.0...v2.0.0)
+
+BREAKING CHANGES
+----------------
+
+ * This package now publishes in ES6 / ES2015 syntax to NPM
+ * Saves access_token and user_id after login for all login types
+   [\#932](https://github.com/matrix-org/matrix-js-sdk/pull/932)
+ * Fix recovery key encoding for base-x 3.0.5
+   [\#931](https://github.com/matrix-org/matrix-js-sdk/pull/931)
+
+Changes in [1.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.2.0) (2019-05-29)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.2.0-rc.1...v1.2.0)
+
+
+Changes in [1.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.2.0-rc.1) (2019-05-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.1.0...v1.2.0-rc.1)
+
+ * interactive-auth now handles requesting email tokens
+   [\#926](https://github.com/matrix-org/matrix-js-sdk/pull/926)
+ * allow access to unreplaced message content
+   [\#923](https://github.com/matrix-org/matrix-js-sdk/pull/923)
+ * Add method to retrieve replacing event
+   [\#922](https://github.com/matrix-org/matrix-js-sdk/pull/922)
+ * More logging when signature verification fails
+   [\#921](https://github.com/matrix-org/matrix-js-sdk/pull/921)
+ * Local echo for m.replace relations
+   [\#920](https://github.com/matrix-org/matrix-js-sdk/pull/920)
+ * Track relations as pending and remove when cancelled
+   [\#919](https://github.com/matrix-org/matrix-js-sdk/pull/919)
+ * Add stringify helper to summarise events when debugging
+   [\#916](https://github.com/matrix-org/matrix-js-sdk/pull/916)
+ * Message editing: filter out replacements for senders that are not the
+   original sender
+   [\#918](https://github.com/matrix-org/matrix-js-sdk/pull/918)
+ * Wait until decrypt before aggregating
+   [\#917](https://github.com/matrix-org/matrix-js-sdk/pull/917)
+ * Message editing: mark original event as replaced instead of replacing the
+   event object
+   [\#914](https://github.com/matrix-org/matrix-js-sdk/pull/914)
+ * Support for replacing message through m.replace relationship.
+   [\#913](https://github.com/matrix-org/matrix-js-sdk/pull/913)
+ * Use a short timeout for .well-known requests
+   [\#912](https://github.com/matrix-org/matrix-js-sdk/pull/912)
+ * Redaction and change events for relations
+   [\#911](https://github.com/matrix-org/matrix-js-sdk/pull/911)
+ * Add basic read path for relations
+   [\#910](https://github.com/matrix-org/matrix-js-sdk/pull/910)
+ * Add a concept of default push rules, using it for tombstone notifications
+   [\#860](https://github.com/matrix-org/matrix-js-sdk/pull/860)
+ * yarn upgrade
+   [\#907](https://github.com/matrix-org/matrix-js-sdk/pull/907)
+
+Changes in [1.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.1.0) (2019-05-07)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.1.0-rc.1...v1.1.0)
+
+ * No Changes since rc.1
+
+Changes in [1.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.1.0-rc.1) (2019-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.4...v1.1.0-rc.1)
+
+ * use the release version of olm 3.1.0
+   [\#903](https://github.com/matrix-org/matrix-js-sdk/pull/903)
+ * Use new Olm repo link in README
+   [\#901](https://github.com/matrix-org/matrix-js-sdk/pull/901)
+ * Support being fed a .well-known config object for validation
+   [\#897](https://github.com/matrix-org/matrix-js-sdk/pull/897)
+ * emit self-membership event at end of handling sync update
+   [\#900](https://github.com/matrix-org/matrix-js-sdk/pull/900)
+ * Use packages.matrix.org for Olm
+   [\#898](https://github.com/matrix-org/matrix-js-sdk/pull/898)
+ * Fix tests on develop
+   [\#899](https://github.com/matrix-org/matrix-js-sdk/pull/899)
+ * Stop syncing when the token is invalid
+   [\#895](https://github.com/matrix-org/matrix-js-sdk/pull/895)
+ * change event redact,  POST request to PUT request
+   [\#887](https://github.com/matrix-org/matrix-js-sdk/pull/887)
+ * Expose better autodiscovery error messages
+   [\#894](https://github.com/matrix-org/matrix-js-sdk/pull/894)
+ * Explicitly guard store usage during sync startup
+   [\#892](https://github.com/matrix-org/matrix-js-sdk/pull/892)
+ * Flag v3 rooms as safe
+   [\#893](https://github.com/matrix-org/matrix-js-sdk/pull/893)
+ * Cache failed capabilities lookups for shorter amounts of time
+   [\#890](https://github.com/matrix-org/matrix-js-sdk/pull/890)
+ * Fix highlight notifications for unencrypted rooms
+   [\#891](https://github.com/matrix-org/matrix-js-sdk/pull/891)
+ * Document checking crypto state before using `hasUnverifiedDevices`
+   [\#889](https://github.com/matrix-org/matrix-js-sdk/pull/889)
+ * Add logging to sync startup path
+   [\#888](https://github.com/matrix-org/matrix-js-sdk/pull/888)
+ * Track e2e highlights better, particularly in 'Mentions Only' rooms
+   [\#886](https://github.com/matrix-org/matrix-js-sdk/pull/886)
+ * support both the incorrect and correct MAC methods
+   [\#882](https://github.com/matrix-org/matrix-js-sdk/pull/882)
+ * Refuse to set forwards pagination token on live timeline
+   [\#885](https://github.com/matrix-org/matrix-js-sdk/pull/885)
+ * Degrade `IndexedDBStore` back to memory only on failure
+   [\#884](https://github.com/matrix-org/matrix-js-sdk/pull/884)
+ * Refuse to link live timelines into the forwards/backwards position when
+   either is invalid
+   [\#877](https://github.com/matrix-org/matrix-js-sdk/pull/877)
+ * Key backup logging improvements
+   [\#883](https://github.com/matrix-org/matrix-js-sdk/pull/883)
+ * Don't assume aborts are always from txn.abort()
+   [\#880](https://github.com/matrix-org/matrix-js-sdk/pull/880)
+ * Add a bunch of logging
+   [\#878](https://github.com/matrix-org/matrix-js-sdk/pull/878)
+ * Refuse splicing the live timeline into a broken position
+   [\#873](https://github.com/matrix-org/matrix-js-sdk/pull/873)
+ * Add existence check to local storage based crypto store
+   [\#872](https://github.com/matrix-org/matrix-js-sdk/pull/872)
+
+Changes in [1.0.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.4) (2019-04-08)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.3...v1.0.4)
+
+ * Hotfix: more logging and potential fixes for timeline corruption issue, see ticket https://github.com/vector-im/riot-web/issues/8593.
+
 Changes in [1.0.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.3) (2019-04-01)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.3-rc.1...v1.0.3)

README.md

@@ -1,8 +1,7 @@
 Matrix Javascript SDK
 =====================
-[![Build Status](http://matrix.org/jenkins/buildStatus/icon?job=JavascriptSDK)](http://matrix.org/jenkins/job/JavascriptSDK/)
 
-This is the [Matrix](https://matrix.org) Client-Server v1/v2 alpha SDK for
+This is the [Matrix](https://matrix.org) Client-Server r0 SDK for
 JavaScript. This SDK can be run in a browser or in Node.js.
 
 Quickstart
@@ -297,9 +296,9 @@ Then visit ``http://localhost:8005`` to see the API docs.
 End-to-end encryption support
 =============================
 
-The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
+The SDK supports end-to-end encryption via the and Megolm protocols, using
-[libolm](http://matrix.org/git/olm). It is left up to the application to make
+[libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the 
-libolm available, via the ``Olm`` global.
+application to make libolm available, via the ``Olm`` global.
 
 It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
 ``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
@@ -318,18 +317,18 @@ specification.
 
 To provide the Olm library in a browser application:
 
- * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
+ * download the transpiled libolm (from https://packages.matrix.org/npm/olm/).
  * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
  
 To provide the Olm library in a node.js application:
 
- * ``yarn add https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``
+ * ``yarn add https://packages.matrix.org/npm/olm/olm-3.0.0.tgz``
    (replace the URL with the latest version you want to use from
-    https://matrix.org/packages/npm/olm/)
+    https://packages.matrix.org/npm/olm/)
  * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
 
 If you want to package Olm as dependency for your node.js application, you can
-use ``yarn add https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``. If your
+use ``yarn add https://packages.matrix.org/npm/olm/olm-3.0.0.tgz``. If your
 application also works without e2e crypto enabled, add ``--optional`` to mark it
 as an optional dependency.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "1.0.3",
+  "version": "2.0.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {
@@ -9,12 +9,12 @@
     "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
     "test": "yarn test:build && yarn test:run",
     "check": "yarn test:build && _mocha --recursive specbuild --colors",
-    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
+    "gendoc": "babel --no-babelrc --plugins transform-class-properties -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
     "start": "yarn start:init && yarn start:watch",
     "start:watch": "babel -s -w --skip-initial-build -d lib src",
     "start:init": "babel -s -d lib src",
     "clean": "rimraf lib dist",
-    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
+    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && terser -c -m -o dist/browser-matrix.min.js --source-map \"content='dist/browser-matrix.js.map'\" dist/browser-matrix.js",
     "dist": "yarn build",
     "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
     "lint": "eslint --max-warnings 101 src spec",
@@ -54,7 +54,6 @@
   "dependencies": {
     "another-json": "^0.2.0",
     "babel-runtime": "^6.26.0",
-    "base-x": "3.0.4",
     "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
@@ -76,6 +75,7 @@
     "browserify-shim": "^3.8.13",
     "eslint": "^5.12.0",
     "eslint-config-google": "^0.7.1",
+    "eslint-plugin-babel": "^5.3.0",
     "exorcist": "^0.4.0",
     "expect": "^1.20.2",
     "istanbul": "^0.4.5",
@@ -84,11 +84,11 @@
     "matrix-mock-request": "^1.2.3",
     "mocha": "^5.2.0",
     "mocha-jenkins-reporter": "^0.4.0",
-    "olm": "https://matrix.org/packages/npm/olm/olm-3.1.0-pre1.tgz",
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.0.tgz",
     "rimraf": "^2.5.4",
     "source-map-support": "^0.4.11",
     "sourceify": "^0.1.0",
-    "uglify-js": "^2.8.26",
+    "terser": "^4.0.0",
     "watchify": "^3.11.1"
   },
   "browserify": {

spec/TestClient.js

@@ -27,6 +27,7 @@ import MockHttpBackend from 'matrix-mock-request';
 import expect from 'expect';
 import Promise from 'bluebird';
 import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';
+import logger from '../src/logger';
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -82,7 +83,7 @@ TestClient.prototype.toString = function() {
  * @return {Promise}
  */
 TestClient.prototype.start = function() {
-    console.log(this + ': starting');
+    logger.log(this + ': starting');
     this.httpBackend.when("GET", "/pushrules").respond(200, {});
     this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
     this.expectDeviceKeyUpload();
@@ -100,7 +101,7 @@ TestClient.prototype.start = function() {
         this.httpBackend.flushAllExpected(),
         testUtils.syncPromise(this.client),
     ]).then(() => {
-        console.log(this + ': started');
+        logger.log(this + ': started');
     });
 };
 
@@ -122,7 +123,7 @@ TestClient.prototype.expectDeviceKeyUpload = function() {
         expect(content.one_time_keys).toBe(undefined);
         expect(content.device_keys).toBeTruthy();
 
-        console.log(self + ': received device keys');
+        logger.log(self + ': received device keys');
         // we expect this to happen before any one-time keys are uploaded.
         expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
 
@@ -159,7 +160,7 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
               expect(content.device_keys).toBe(undefined);
               expect(content.one_time_keys).toBeTruthy();
               expect(content.one_time_keys).toNotEqual({});
-              console.log('%s: received %i one-time keys', this,
+              logger.log('%s: received %i one-time keys', this,
                           Object.keys(content.one_time_keys).length);
               this.oneTimeKeys = content.one_time_keys;
               return {one_time_key_counts: {
@@ -223,11 +224,11 @@ TestClient.prototype.getSigningKey = function() {
  * @returns {Promise} promise which completes once the sync has been flushed
  */
 TestClient.prototype.flushSync = function() {
-    console.log(`${this}: flushSync`);
+    logger.log(`${this}: flushSync`);
     return Promise.all([
         this.httpBackend.flush('/sync', 1),
         testUtils.syncPromise(this.client),
     ]).then(() => {
-        console.log(`${this}: flushSync completed`);
+        logger.log(`${this}: flushSync completed`);
     });
 };

spec/integ/devicelist-integ-spec.js

@@ -20,6 +20,7 @@ import Promise from 'bluebird';
 
 import TestClient from '../TestClient';
 import testUtils from '../test-utils';
+import logger from '../../src/logger';
 
 const ROOM_ID = "!room:id";
 
@@ -71,7 +72,7 @@ function getSyncResponse(roomMembers) {
 
 describe("DeviceList management:", function() {
     if (!global.Olm) {
-        console.warn('not running deviceList tests: Olm not present');
+        logger.warn('not running deviceList tests: Olm not present');
         return;
     }
 
@@ -87,7 +88,7 @@ describe("DeviceList management:", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         // we create our own sessionStoreBackend so that we can use it for
         // another TestClient.
@@ -108,7 +109,7 @@ describe("DeviceList management:", function() {
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log("Forcing alice to download our device keys");
+            logger.log("Forcing alice to download our device keys");
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
                 device_keys: {
@@ -121,7 +122,7 @@ describe("DeviceList management:", function() {
                 aliceTestClient.httpBackend.flush('/keys/query', 1),
             ]);
         }).then(function() {
-            console.log("Telling alice to send a megolm message");
+            logger.log("Telling alice to send a megolm message");
 
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',

spec/integ/matrix-client-crypto.spec.js

@@ -36,6 +36,7 @@ import Promise from 'bluebird';
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
 const TestClient = require('../TestClient').default;
+import logger from '../../src/logger';
 
 let aliTestClient;
 const roomId = "!room:localhost";
@@ -95,7 +96,7 @@ function expectBobQueryKeys() {
 
     const aliKeys = {};
     aliKeys[aliDeviceId] = aliTestClient.deviceKeys;
-    console.log("query result will be", aliKeys);
+    logger.log("query result will be", aliKeys);
 
     bobTestClient.httpBackend.when(
         "POST", "/keys/query",
@@ -334,7 +335,7 @@ function recvMessage(httpBackend, client, sender, message) {
             if (event.getType() == "m.room.member") {
                 return;
             }
-            console.log(client.credentials.userId + " received event",
+            logger.log(client.credentials.userId + " received event",
                         event);
 
             client.removeListener("event", onEvent);
@@ -405,7 +406,7 @@ describe("MatrixClient crypto", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
         await aliTestClient.client.initCrypto();
@@ -607,7 +608,7 @@ describe("MatrixClient crypto", function() {
 
                 const eventPromise = new Promise((resolve, reject) => {
                     const onEvent = function(event) {
-                        console.log(bobUserId + " received event",
+                        logger.log(bobUserId + " received event",
                                     event);
                         resolve(event);
                     };
@@ -734,7 +735,7 @@ describe("MatrixClient crypto", function() {
 
         return Promise.resolve()
             .then(() => {
-                console.log(aliTestClient + ': starting');
+                logger.log(aliTestClient + ': starting');
                 httpBackend.when("GET", "/pushrules").respond(200, {});
                 httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
                 aliTestClient.expectDeviceKeyUpload();
@@ -746,7 +747,7 @@ describe("MatrixClient crypto", function() {
                 aliTestClient.client.startClient({});
 
                 return httpBackend.flushAllExpected().then(() => {
-                    console.log(aliTestClient + ': started');
+                    logger.log(aliTestClient + ': started');
                 });
             })
             .then(() => httpBackend.when("POST", "/keys/upload")
@@ -755,7 +756,7 @@ describe("MatrixClient crypto", function() {
                     expect(content.one_time_keys).toNotEqual({});
                     expect(Object.keys(content.one_time_keys).length)
                         .toBeGreaterThanOrEqualTo(1);
-                    console.log('received %i one-time keys',
+                    logger.log('received %i one-time keys',
                                 Object.keys(content.one_time_keys).length);
                     // cancel futher calls by telling the client
                     // we have more than we need

spec/integ/matrix-client-event-emitter.spec.js

@@ -15,7 +15,7 @@ describe("MatrixClient events", function() {
     const selfAccessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
         client = sdk.createClient({
@@ -157,7 +157,7 @@ describe("MatrixClient events", function() {
                     return;
                 }
 
-                expect(event.event).toEqual(SYNC_DATA.presence.events[0]);
+                expect(event.event).toMatch(SYNC_DATA.presence.events[0]);
                 expect(user.presence).toEqual(
                     SYNC_DATA.presence.events[0].content.presence,
                 );

spec/integ/matrix-client-event-timeline.spec.js

@@ -5,6 +5,7 @@ const sdk = require("../..");
 const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const EventTimeline = sdk.EventTimeline;
+import logger from '../../src/logger';
 
 const baseUrl = "http://localhost.or.something";
 const userId = "@alice:localhost";
@@ -84,7 +85,7 @@ function startClient(httpBackend, client) {
     // set up a promise which will resolve once the client is initialised
     const deferred = Promise.defer();
     client.on("sync", function(state) {
-        console.log("sync", state);
+        logger.log("sync", state);
         if (state != "SYNCING") {
             return;
         }
@@ -102,7 +103,7 @@ describe("getEventTimeline support", function() {
     let client;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
     });
@@ -227,7 +228,7 @@ describe("MatrixClient event timelines", function() {
     let httpBackend = null;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
 
@@ -669,11 +670,11 @@ describe("MatrixClient event timelines", function() {
                 // initiate the send, and set up checks to be done when it completes
                 // - but note that it won't complete until after the /sync does, below.
                 client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                    console.log("sendTextMessage completed");
+                    logger.log("sendTextMessage completed");
                     expect(res.event_id).toEqual(event.event_id);
                     return client.getEventTimeline(timelineSet, event.event_id);
                 }).then(function(tl) {
-                    console.log("getEventTimeline completed (2)");
+                    logger.log("getEventTimeline completed (2)");
                     expect(tl.getEvents().length).toEqual(2);
                     expect(tl.getEvents()[1].getContent().body).toEqual("a body");
                 }),
@@ -684,7 +685,7 @@ describe("MatrixClient event timelines", function() {
                 ]).then(function() {
                     return client.getEventTimeline(timelineSet, event.event_id);
                 }).then(function(tl) {
-                    console.log("getEventTimeline completed (1)");
+                    logger.log("getEventTimeline completed (1)");
                     expect(tl.getEvents().length).toEqual(2);
                     expect(tl.getEvents()[1].event).toEqual(event);
 

spec/integ/matrix-client-methods.spec.js

@@ -21,7 +21,7 @@ describe("MatrixClient", function() {
     const accessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         store = new MemoryStore();
 
@@ -355,9 +355,9 @@ describe("MatrixClient", function() {
                 return client._crypto._olmDevice.sign(anotherjson.stringify(b));
             };
 
-            console.log("Ed25519: " + ed25519key);
+            logger.log("Ed25519: " + ed25519key);
-            console.log("boris:", sign(borisKeys.dev1));
+            logger.log("boris:", sign(borisKeys.dev1));
-            console.log("chaz:", sign(chazKeys.dev2));
+            logger.log("chaz:", sign(chazKeys.dev2));
             */
 
             httpBackend.when("POST", "/keys/query").check(function(req) {

spec/integ/matrix-client-opts.spec.js

@@ -58,7 +58,7 @@ describe("MatrixClient opts", function() {
     };
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
     });
 

spec/integ/matrix-client-retrying.spec.js

@@ -20,7 +20,7 @@ describe("MatrixClient retrying", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
         scheduler = new sdk.MatrixScheduler();

spec/integ/matrix-client-room-timeline.spec.js

@@ -104,7 +104,7 @@ describe("MatrixClient room timelines", function() {
     }
 
     beforeEach(function(done) {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
         client = sdk.createClient({

spec/integ/matrix-client-syncing.spec.js

@@ -23,7 +23,7 @@ describe("MatrixClient syncing", function() {
     const roomTwo = "!bar:localhost";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
         client = sdk.createClient({

spec/integ/megolm-integ.spec.js

@@ -23,6 +23,7 @@ import expect from 'expect';
 const utils = require('../../lib/utils');
 const testUtils = require('../test-utils');
 const TestClient = require('../TestClient').default;
+import logger from '../../src/logger';
 
 const ROOM_ID = "!room:id";
 
@@ -203,7 +204,7 @@ function getSyncResponse(roomMembers) {
 
 describe("megolm", function() {
     if (!global.Olm) {
-        console.warn('not running megolm tests: Olm not present');
+        logger.warn('not running megolm tests: Olm not present');
         return;
     }
     const Olm = global.Olm;
@@ -282,7 +283,7 @@ describe("megolm", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         aliceTestClient = new TestClient(
             "@alice:localhost", "xzcvb", "akjgkrgjs",
@@ -416,7 +417,7 @@ describe("megolm", function() {
 
             return new Promise((resolve, reject) => {
                 event.once('Event.decrypted', (ev) => {
-                    console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
                     resolve(ev);
                 });
             });
@@ -555,7 +556,7 @@ describe("megolm", function() {
             ).respond(200, function(path, content) {
                 const ct = content.ciphertext;
                 const r = inboundGroupSession.decrypt(ct);
-                console.log('Decrypted received megolm message', r);
+                logger.log('Decrypted received megolm message', r);
 
                 expect(r.message_index).toEqual(0);
                 const decrypted = JSON.parse(r.plaintext);
@@ -600,7 +601,7 @@ describe("megolm", function() {
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log('Forcing alice to download our device keys');
+            logger.log('Forcing alice to download our device keys');
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                 200, getTestKeysQueryResponse('@bob:xyz'),
@@ -611,10 +612,10 @@ describe("megolm", function() {
                 aliceTestClient.httpBackend.flush('/keys/query', 1),
             ]);
         }).then(function() {
-            console.log('Telling alice to block our device');
+            logger.log('Telling alice to block our device');
             aliceTestClient.client.setDeviceBlocked('@bob:xyz', 'DEVICE_ID');
 
-            console.log('Telling alice to send a megolm message');
+            logger.log('Telling alice to send a megolm message');
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, {
@@ -656,7 +657,7 @@ describe("megolm", function() {
 
             return aliceTestClient.flushSync();
         }).then(function() {
-            console.log("Fetching bob's devices and marking known");
+            logger.log("Fetching bob's devices and marking known");
 
             aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                 200, getTestKeysQueryResponse('@bob:xyz'),
@@ -669,17 +670,17 @@ describe("megolm", function() {
                 aliceTestClient.client.setDeviceKnown('@bob:xyz', 'DEVICE_ID');
             });
         }).then(function() {
-            console.log('Telling alice to send a megolm message');
+            logger.log('Telling alice to send a megolm message');
 
             aliceTestClient.httpBackend.when(
                 'PUT', '/sendToDevice/m.room.encrypted/',
             ).respond(200, function(path, content) {
-                console.log('sendToDevice: ', content);
+                logger.log('sendToDevice: ', content);
                 const m = content.messages['@bob:xyz'].DEVICE_ID;
                 const ct = m.ciphertext[testSenderKey];
                 expect(ct.type).toEqual(1); // normal message
                 const decrypted = JSON.parse(p2pSession.decrypt(ct.type, ct.body));
-                console.log('decrypted sendToDevice:', decrypted);
+                logger.log('decrypted sendToDevice:', decrypted);
                 expect(decrypted.type).toEqual('m.room_key');
                 megolmSessionId = decrypted.content.session_id;
                 return {};
@@ -688,7 +689,7 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
-                console.log('/send:', content);
+                logger.log('/send:', content);
                 expect(content.session_id).toEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
@@ -704,14 +705,14 @@ describe("megolm", function() {
                 }),
             ]);
         }).then(function() {
-            console.log('Telling alice to block our device');
+            logger.log('Telling alice to block our device');
             aliceTestClient.client.setDeviceBlocked('@bob:xyz', 'DEVICE_ID');
 
-            console.log('Telling alice to send another megolm message');
+            logger.log('Telling alice to send another megolm message');
             aliceTestClient.httpBackend.when(
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
-                console.log('/send:', content);
+                logger.log('/send:', content);
                 expect(content.session_id).toNotEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
@@ -792,7 +793,7 @@ describe("megolm", function() {
             aliceTestClient.httpBackend.when(
                 'PUT', '/sendToDevice/m.room.encrypted/',
             ).respond(200, function(path, content) {
-                console.log("sendToDevice: ", content);
+                logger.log("sendToDevice: ", content);
                 const m = content.messages[aliceTestClient.userId].DEVICE_ID;
                 const ct = m.ciphertext[testSenderKey];
                 expect(ct.type).toEqual(0); // pre-key message
@@ -812,7 +813,7 @@ describe("megolm", function() {
             ).respond(200, function(path, content) {
                 const ct = content.ciphertext;
                 const r = inboundGroupSession.decrypt(ct);
-                console.log('Decrypted received megolm message', r);
+                logger.log('Decrypted received megolm message', r);
                 decrypted = JSON.parse(r.plaintext);
 
                 return {
@@ -865,7 +866,7 @@ describe("megolm", function() {
             return aliceTestClient.flushSync();
         }).then(function() {
             // this will block
-            console.log('Forcing alice to download our device keys');
+            logger.log('Forcing alice to download our device keys');
             downloadPromise = aliceTestClient.client.downloadKeys(['@bob:xyz']);
 
             // so will this.

spec/olm-loader.js

@@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// try to load the olm library.
+import logger from '../src/logger';
 
+// try to load the olm library.
 try {
     global.Olm = require('olm');
-    console.log('loaded libolm');
+    logger.log('loaded libolm');
 } catch (e) {
-    console.warn("unable to run crypto tests: libolm not available");
+    logger.warn("unable to run crypto tests: libolm not available");
 }

spec/test-utils.js

@@ -6,6 +6,7 @@ const logger = require("../lib/logger");
 // load olm before the sdk if possible
 import './olm-loader';
 
+import logger from '../src/logger';
 import sdk from '..';
 const MatrixEvent = sdk.MatrixEvent;
 
@@ -26,7 +27,7 @@ module.exports.syncPromise = function(client, count) {
 
     const p = new Promise((resolve, reject) => {
         const cb = (state) => {
-            console.log(`${Date.now()} syncPromise(${count}): ${state}`);
+            logger.log(`${Date.now()} syncPromise(${count}): ${state}`);
             if (state == 'SYNCING') {
                 resolve();
             } else {
@@ -49,8 +50,8 @@ module.exports.syncPromise = function(client, count) {
 module.exports.beforeEach = function(context) {
     const desc = context.currentTest.fullTitle();
 
-    console.log(desc);
+    logger.log(desc);
-    console.log(new Array(1 + desc.length).join("="));
+    logger.log(new Array(1 + desc.length).join("="));
 };
 
 /**
@@ -233,11 +234,11 @@ module.exports.awaitDecryption = function(event) {
         return Promise.resolve(event);
     }
 
-    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+    logger.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
 
     return new Promise((resolve, reject) => {
         event.once('Event.decrypted', (ev) => {
-            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            logger.log(`${Date.now()} event ${event.getId()} now decrypted`);
             resolve(ev);
         });
     });

spec/unit/autodiscovery.spec.js

@@ -30,7 +30,7 @@ describe("AutoDiscovery", function() {
     let httpBackend = null;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new MockHttpBackend();
         sdk.request(httpBackend.requestFn);
     });
@@ -94,7 +94,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -117,7 +117,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -140,7 +140,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -163,7 +163,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -191,7 +191,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -217,7 +217,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_PROMPT",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -245,7 +245,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -274,7 +274,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -303,7 +303,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -334,7 +334,7 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid homeserver discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_HOMESERVER,
                         base_url: null,
                     },
                     "m.identity_server": {
@@ -439,14 +439,14 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
                         base_url: null,
                     },
                 };
@@ -479,14 +479,14 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
                         base_url: null,
                     },
                 };
@@ -520,14 +520,14 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
                         base_url: null,
                     },
                 };
@@ -561,14 +561,14 @@ describe("AutoDiscovery", function() {
                 const expected = {
                     "m.homeserver": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IS,
 
                         // We still expect the base_url to be here for debugging purposes
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
                         state: "FAIL_ERROR",
-                        error: "Invalid identity server discovery response",
+                        error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
                         base_url: null,
                     },
                 };

spec/unit/content-repo.spec.js

@@ -9,7 +9,7 @@ describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     describe("getHttpUriForMxc", function() {

spec/unit/crypto/DeviceList.spec.js

@@ -19,6 +19,7 @@ import DeviceList from '../../../lib/crypto/DeviceList';
 import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
 import testUtils from '../../test-utils';
 import utils from '../../../lib/utils';
+import logger from '../../../src/logger';
 
 import expect from 'expect';
 import Promise from 'bluebird';
@@ -59,7 +60,7 @@ describe('DeviceList', function() {
     let deviceLists = [];
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         deviceLists = [];
 
@@ -134,7 +135,7 @@ describe('DeviceList', function() {
         }).then(() => {
             // uh-oh; user restarts before second request completes. The new instance
             // should know we never got a complete device list.
-            console.log("Creating new devicelist to simulate app reload");
+            logger.log("Creating new devicelist to simulate app reload");
             downloadSpy.reset();
             const dl2 = createTestDeviceList();
             const queryDefer3 = Promise.defer();

spec/unit/crypto/algorithms/megolm.spec.js

@@ -10,6 +10,7 @@ import MockStorageApi from '../../../MockStorageApi';
 import testUtils from '../../../test-utils';
 import OlmDevice from '../../../../lib/crypto/OlmDevice';
 import Crypto from '../../../../lib/crypto';
+import logger from '../../../../src/logger';
 
 const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
@@ -21,7 +22,7 @@ const Olm = global.Olm;
 
 describe("MegolmDecryption", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm unit tests: libolm not present');
+        logger.warn('Not running megolm unit tests: libolm not present');
         return;
     }
 
@@ -31,7 +32,7 @@ describe("MegolmDecryption", function() {
     let mockBaseApis;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         await Olm.init();
 

spec/unit/crypto/algorithms/olm.spec.js

@@ -20,6 +20,7 @@ import expect from 'expect';
 import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
 import MockStorageApi from '../../../MockStorageApi';
 import testUtils from '../../../test-utils';
+import logger from '../../../../src/logger';
 
 import OlmDevice from '../../../../lib/crypto/OlmDevice';
 import olmlib from '../../../../lib/crypto/olmlib';
@@ -45,7 +46,7 @@ async function setupSession(initiator, opponent) {
 
 describe("OlmDecryption", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm unit tests: libolm not present');
+        logger.warn('Not running megolm unit tests: libolm not present');
         return;
     }
 
@@ -53,7 +54,7 @@ describe("OlmDecryption", function() {
     let bobOlmDevice;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         await global.Olm.init();
 

spec/unit/crypto/backup.spec.js

@@ -28,6 +28,7 @@ import testUtils from '../../test-utils';
 
 import OlmDevice from '../../../lib/crypto/OlmDevice';
 import Crypto from '../../../lib/crypto';
+import logger from '../../../src/logger';
 
 const Olm = global.Olm;
 
@@ -112,7 +113,7 @@ function makeTestClient(sessionStore, cryptoStore) {
 
 describe("MegolmBackup", function() {
     if (!global.Olm) {
-        console.warn('Not running megolm backup unit tests: libolm not present');
+        logger.warn('Not running megolm backup unit tests: libolm not present');
         return;
     }
 
@@ -125,7 +126,7 @@ describe("MegolmBackup", function() {
     let megolmDecryption;
     beforeEach(async function() {
         await Olm.init();
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
         mockCrypto.backupKey = new Olm.PkEncryption();

spec/unit/crypto/verification/qr_code.spec.js

@@ -13,15 +13,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import logger from '../../../../src/logger';
 
 try {
     global.Olm = require('olm');
 } catch (e) {
-    console.warn("unable to run device verification tests: libolm not available");
+    logger.warn("unable to run device verification tests: libolm not available");
 }
 
 import expect from 'expect';
-
 import DeviceInfo from '../../../../lib/crypto/deviceinfo';
 
 import {ShowQRCode, ScanQRCode} from '../../../../lib/crypto/verification/QRCode';
@@ -30,7 +30,7 @@ const Olm = global.Olm;
 
 describe("QR code verification", function() {
     if (!global.Olm) {
-        console.warn('Not running device verification tests: libolm not present');
+        logger.warn('Not running device verification tests: libolm not present');
         return;
     }
 

spec/unit/crypto/verification/request.spec.js

@@ -13,11 +13,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import logger from '../../../../src/logger';
 
 try {
     global.Olm = require('olm');
 } catch (e) {
-    console.warn("unable to run device verification tests: libolm not available");
+    logger.warn("unable to run device verification tests: libolm not available");
 }
 
 import expect from 'expect';
@@ -32,7 +33,7 @@ import {makeTestClients} from './util';
 
 describe("verification request", function() {
     if (!global.Olm) {
-        console.warn('Not running device verification unit tests: libolm not present');
+        logger.warn('Not running device verification unit tests: libolm not present');
         return;
     }
 

spec/unit/crypto/verification/sas.spec.js

@@ -13,11 +13,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import logger from '../../../../src/logger';
 
 try {
     global.Olm = require('olm');
 } catch (e) {
-    console.warn("unable to run device verification tests: libolm not available");
+    logger.warn("unable to run device verification tests: libolm not available");
 }
 
 import expect from 'expect';
@@ -37,7 +38,7 @@ import {makeTestClients} from './util';
 
 describe("SAS verification", function() {
     if (!global.Olm) {
-        console.warn('Not running device verification unit tests: libolm not present');
+        logger.warn('Not running device verification unit tests: libolm not present');
         return;
     }
 
@@ -58,8 +59,16 @@ describe("SAS verification", function() {
         expect(spy).toHaveBeenCalled();
     });
 
-    it("should verify a key", async function() {
+    describe("verification", function() {
-        const [alice, bob] = await makeTestClients(
+        let alice;
+        let bob;
+        let aliceSasEvent;
+        let bobSasEvent;
+        let aliceVerifier;
+        let bobPromise;
+
+        beforeEach(async function() {
+            [alice, bob] = await makeTestClients(
                 [
                     {userId: "@alice:example.com", deviceId: "Osborne2"},
                     {userId: "@bob:example.com", deviceId: "Dynabook"},
@@ -105,10 +114,10 @@ describe("SAS verification", function() {
                 return Promise.resolve();
             };
 
-        let aliceSasEvent;
+            aliceSasEvent = null;
-        let bobSasEvent;
+            bobSasEvent = null;
 
-        const bobPromise = new Promise((resolve, reject) => {
+            bobPromise = new Promise((resolve, reject) => {
                 bob.on("crypto.verification.start", (verifier) => {
                     verifier.on("show_sas", (e) => {
                         if (!e.sas.emoji || !e.sas.decimal) {
@@ -130,7 +139,7 @@ describe("SAS verification", function() {
                 });
             });
 
-        const aliceVerifier = alice.beginKeyVerification(
+            aliceVerifier = alice.beginKeyVerification(
                 verificationMethods.SAS, bob.getUserId(), bob.deviceId,
             );
             aliceVerifier.on("show_sas", (e) => {
@@ -149,16 +158,73 @@ describe("SAS verification", function() {
                     }
                 }
             });
+        });
+
+        it("should verify a key", async function() {
+            let macMethod;
+            const origSendToDevice = alice.sendToDevice;
+            bob.sendToDevice = function(type, map) {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.getUserId()][alice.deviceId]
+                        .message_authentication_code;
+                }
+                return origSendToDevice.call(this, type, map);
+            };
+
             await Promise.all([
                 aliceVerifier.verify(),
                 bobPromise.then((verifier) => verifier.verify()),
             ]);
+
+            // make sure that it uses the preferred method
+            expect(macMethod).toBe("hkdf-hmac-sha256");
+
+            // make sure Alice and Bob verified each other
             expect(alice.setDeviceVerified)
                 .toHaveBeenCalledWith(bob.getUserId(), bob.deviceId);
             expect(bob.setDeviceVerified)
                 .toHaveBeenCalledWith(alice.getUserId(), alice.deviceId);
         });
 
+        it("should be able to verify using the old MAC", async function() {
+            // pretend that Alice can only understand the old (incorrect) MAC,
+            // and make sure that she can still verify with Bob
+            let macMethod;
+            const origSendToDevice = alice.sendToDevice;
+            alice.sendToDevice = function(type, map) {
+                if (type === "m.key.verification.start") {
+                    // Note: this modifies not only the message that Bob
+                    // receives, but also the copy of the message that Alice
+                    // has, since it is the same object.  If this does not
+                    // happen, the verification will fail due to a hash
+                    // commitment mismatch.
+                    map[bob.getUserId()][bob.deviceId]
+                        .message_authentication_codes = ['hmac-sha256'];
+                }
+                return origSendToDevice.call(this, type, map);
+            };
+            bob.sendToDevice = function(type, map) {
+                if (type === "m.key.verification.accept") {
+                    macMethod = map[alice.getUserId()][alice.deviceId]
+                        .message_authentication_code;
+                }
+                return origSendToDevice.call(this, type, map);
+            };
+
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => verifier.verify()),
+            ]);
+
+            expect(macMethod).toBe("hmac-sha256");
+
+            expect(alice.setDeviceVerified)
+                .toHaveBeenCalledWith(bob.getUserId(), bob.deviceId);
+            expect(bob.setDeviceVerified)
+                .toHaveBeenCalledWith(alice.getUserId(), alice.deviceId);
+        });
+    });
+
     it("should send a cancellation message on error", async function() {
         const [alice, bob] = await makeTestClients(
             [

spec/unit/crypto/verification/util.js

@@ -29,7 +29,7 @@ export async function makeTestClients(userInfos, options) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
                     if (deviceId in clientMap[userId]) {
                         const event = new MatrixEvent({
-                            sender: this.getUserId(), // eslint-disable-line no-invalid-this
+                            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
                             type: type,
                             content: msg,
                         });

spec/unit/event-timeline.spec.js

@@ -18,7 +18,7 @@ describe("EventTimeline", function() {
     let timeline;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
 
         // XXX: this is a horrid hack; should use sinon or something instead to mock
         const timelineSet = { room: { roomId: roomId }};

spec/unit/event.spec.js

@@ -21,10 +21,11 @@ import testUtils from '../test-utils';
 
 import expect from 'expect';
 import Promise from 'bluebird';
+import logger from '../../src/logger';
 
 describe("MatrixEvent", () => {
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     describe(".attemptDecryption", () => {
@@ -48,7 +49,7 @@ describe("MatrixEvent", () => {
             const crypto = {
                 decryptEvent: function() {
                     ++callCount;
-                    console.log(`decrypt: ${callCount}`);
+                    logger.log(`decrypt: ${callCount}`);
                     if (callCount == 1) {
                         // schedule a second decryption attempt while
                         // the first one is still running.

spec/unit/filter.spec.js

@@ -12,7 +12,7 @@ describe("Filter", function() {
     let filter;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         filter = new Filter(userId);
     });
 

spec/unit/interactive-auth.spec.js

@@ -24,6 +24,7 @@ const InteractiveAuth = sdk.InteractiveAuth;
 const MatrixError = sdk.MatrixError;
 
 import expect from 'expect';
+import logger from '../../src/logger';
 
 // Trivial client object to test interactive auth
 // (we do not need TestClient here)
@@ -35,7 +36,7 @@ class FakeClient {
 
 describe("InteractiveAuth", function() {
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     it("should start an auth stage and complete it", function(done) {
@@ -64,7 +65,7 @@ describe("InteractiveAuth", function() {
 
         // first we expect a call here
         stateUpdated.andCall(function(stage) {
-            console.log('aaaa');
+            logger.log('aaaa');
             expect(stage).toEqual("logintype");
             ia.submitAuthDict({
                 type: "logintype",
@@ -75,7 +76,7 @@ describe("InteractiveAuth", function() {
         // .. which should trigger a call here
         const requestRes = {"a": "b"};
         doRequest.andCall(function(authData) {
-            console.log('cccc');
+            logger.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
                 type: "logintype",
@@ -106,7 +107,7 @@ describe("InteractiveAuth", function() {
 
         // first we expect a call to doRequest
         doRequest.andCall(function(authData) {
-            console.log("request1", authData);
+            logger.log("request1", authData);
             expect(authData).toEqual({});
             const err = new MatrixError({
                 session: "sessionId",
@@ -132,7 +133,7 @@ describe("InteractiveAuth", function() {
 
             // submitAuthDict should trigger another call to doRequest
             doRequest.andCall(function(authData) {
-                console.log("request2", authData);
+                logger.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
                     type: "logintype",

spec/unit/login.spec.js

@@ -0,0 +1,25 @@
+import expect from 'expect';
+import TestClient from '../TestClient';
+
+describe('Login request', function() {
+    let client;
+
+    beforeEach(function() {
+        client = new TestClient();
+    });
+
+    afterEach(function() {
+        client.stop();
+    });
+
+    it('should store "access_token" and "user_id" if in response', async function() {
+        const response = { user_id: 1, access_token: Date.now().toString(16) };
+
+        client.httpBackend.when('POST', '/login').respond(200, response);
+        client.httpBackend.flush('/login', 1, 100);
+        await client.client.login('m.login.any', { user: 'test', password: '12312za' });
+
+        expect(client.client.getAccessToken()).toBe(response.access_token);
+        expect(client.client.getUserId()).toBe(response.user_id);
+    });
+});

spec/unit/matrix-client.spec.js

@@ -7,6 +7,7 @@ const utils = require("../test-utils");
 
 import expect from 'expect';
 import lolex from 'lolex';
+import logger from '../../src/logger';
 
 describe("MatrixClient", function() {
     const userId = "@alice:bar";
@@ -69,7 +70,7 @@ describe("MatrixClient", function() {
             "MatrixClient[UT] RECV " + method + " " + path + "  " +
             "EXPECT " + (next ? next.method : next) + " " + (next ? next.path : next)
         );
-        console.log(logLine);
+        logger.log(logLine);
 
         if (!next) { // no more things to return
             if (pendingLookup) {
@@ -91,7 +92,7 @@ describe("MatrixClient", function() {
             return pendingLookup.promise;
         }
         if (next.path === path && next.method === method) {
-            console.log(
+            logger.log(
                 "MatrixClient[UT] Matched. Returning " +
                 (next.error ? "BAD" : "GOOD") + " response",
             );
@@ -124,7 +125,7 @@ describe("MatrixClient", function() {
     }
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         clock = lolex.install();
         scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
@@ -353,7 +354,7 @@ describe("MatrixClient", function() {
         function syncChecker(expectedStates, done) {
             return function syncListener(state, old) {
                 const expected = expectedStates.shift();
-                console.log(
+                logger.log(
                     "'sync' curr=%s old=%s EXPECT=%s", state, old, expected,
                 );
                 if (!expected) {

spec/unit/realtime-callbacks.spec.js

@@ -15,7 +15,7 @@ describe("realtime-callbacks", function() {
     }
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         clock = lolex.install();
         const fakeDate = clock.Date;
         callbacks.setNow(fakeDate.now.bind(fakeDate));
@@ -56,8 +56,8 @@ describe("realtime-callbacks", function() {
         it("should set 'this' to the global object", function() {
             let passed = false;
             const callback = function() {
-                expect(this).toBe(global); // eslint-disable-line no-invalid-this
+                expect(this).toBe(global); // eslint-disable-line babel/no-invalid-this
-                expect(this.console).toBeTruthy(); // eslint-disable-line no-invalid-this
+                expect(this.console).toBeTruthy(); // eslint-disable-line babel/no-invalid-this
                 passed = true;
             };
             callbacks.setTimeout(callback);

spec/unit/room-member.spec.js

@@ -14,7 +14,7 @@ describe("RoomMember", function() {
     let member;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         member = new RoomMember(roomId, userA);
     });
 

spec/unit/room-state.spec.js

@@ -17,7 +17,7 @@ describe("RoomState", function() {
     let state;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         state = new RoomState(roomId);
         state.setStateEvents([
             utils.mkMembership({  // userA joined

spec/unit/room.spec.js

@@ -19,7 +19,7 @@ describe("Room", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         room = new Room(roomId);
         // mock RoomStates
         room.oldState = room.getLiveTimeline()._startState =
@@ -1318,6 +1318,9 @@ describe("Room", function() {
                     // events should already be MatrixEvents
                     return function(event) {return event;};
                 },
+                isCryptoEnabled() {
+                    return true;
+                },
                 isRoomEncrypted: function() {
                     return false;
                 },

spec/unit/scheduler.spec.js

@@ -26,7 +26,7 @@ describe("MatrixScheduler", function() {
     });
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         clock = lolex.install();
         scheduler = new MatrixScheduler(function(ev, attempts, err) {
             if (retryFn) {

spec/unit/sync-accumulator.spec.js

@@ -26,7 +26,7 @@ describe("SyncAccumulator", function() {
     let sa;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         sa = new SyncAccumulator({
             maxTimelineEntries: 10,
         });

spec/unit/timeline-window.spec.js

@@ -68,7 +68,7 @@ function createLinkedTimelines() {
 
 describe("TimelineIndex", function() {
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     describe("minIndex", function() {
@@ -164,7 +164,7 @@ describe("TimelineWindow", function() {
     }
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     describe("load", function() {

spec/unit/user.spec.js

@@ -11,7 +11,7 @@ describe("User", function() {
     let user;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         user = new User(userId);
     });
 

spec/unit/utils.spec.js

@@ -7,7 +7,7 @@ import expect from 'expect';
 
 describe("utils", function() {
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
     });
 
     describe("encodeParams", function() {

src/autodiscovery.js

@@ -17,7 +17,7 @@ limitations under the License.
 /** @module auto-discovery */
 
 import Promise from 'bluebird';
-const logger = require("./logger");
+import logger from './logger';
 import { URL as NodeURL } from "url";
 
 // Dev note: Auto discovery is part of the spec.
@@ -102,6 +102,56 @@ export class AutoDiscovery {
     // translate the meaning of the states in the spec, but also
     // support our own if needed.
 
+    static get ERROR_INVALID() {
+        return "Invalid homeserver discovery response";
+    }
+
+    static get ERROR_GENERIC_FAILURE() {
+        return "Failed to get autodiscovery configuration from server";
+    }
+
+    static get ERROR_INVALID_HS_BASE_URL() {
+        return "Invalid base_url for m.homeserver";
+    }
+
+    static get ERROR_INVALID_HOMESERVER() {
+        return "Homeserver URL does not appear to be a valid Matrix homeserver";
+    }
+
+    static get ERROR_INVALID_IS_BASE_URL() {
+        return "Invalid base_url for m.identity_server";
+    }
+
+    static get ERROR_INVALID_IDENTITY_SERVER() {
+        return "Identity server URL does not appear to be a valid identity server";
+    }
+
+    static get ERROR_INVALID_IS() {
+        return "Invalid identity server discovery response";
+    }
+
+    static get ERROR_MISSING_WELLKNOWN() {
+        return "No .well-known JSON file found";
+    }
+
+    static get ERROR_INVALID_JSON() {
+        return "Invalid JSON";
+    }
+
+    static get ALL_ERRORS() {
+        return [
+            AutoDiscovery.ERROR_INVALID,
+            AutoDiscovery.ERROR_GENERIC_FAILURE,
+            AutoDiscovery.ERROR_INVALID_HS_BASE_URL,
+            AutoDiscovery.ERROR_INVALID_HOMESERVER,
+            AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
+            AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
+            AutoDiscovery.ERROR_INVALID_IS,
+            AutoDiscovery.ERROR_MISSING_WELLKNOWN,
+            AutoDiscovery.ERROR_INVALID_JSON,
+        ];
+    }
+
     /**
      * The auto discovery failed. The client is expected to communicate
      * the error to the user and refuse logging in.
@@ -137,6 +187,166 @@ export class AutoDiscovery {
      */
     static get SUCCESS() { return "SUCCESS"; }
 
+    /**
+     * Validates and verifies client configuration information for purposes
+     * of logging in. Such information includes the homeserver URL
+     * and identity server URL the client would want. Additional details
+     * may also be included, and will be transparently brought into the
+     * response object unaltered.
+     * @param {string} wellknown The configuration object itself, as returned
+     * by the .well-known auto-discovery endpoint.
+     * @return {Promise<DiscoveredClientConfig>} Resolves to the verified
+     * configuration, which may include error states. Rejects on unexpected
+     * failure, not when verification fails.
+     */
+    static async fromDiscoveryConfig(wellknown) {
+        // Step 1 is to get the config, which is provided to us here.
+
+        // We default to an error state to make the first few checks easier to
+        // write. We'll update the properties of this object over the duration
+        // of this function.
+        const clientConfig = {
+            "m.homeserver": {
+                state: AutoDiscovery.FAIL_ERROR,
+                error: AutoDiscovery.ERROR_INVALID,
+                base_url: null,
+            },
+            "m.identity_server": {
+                // Technically, we don't have a problem with the identity server
+                // config at this point.
+                state: AutoDiscovery.PROMPT,
+                error: null,
+                base_url: null,
+            },
+        };
+
+        if (!wellknown || !wellknown["m.homeserver"]) {
+            logger.error("No m.homeserver key in config");
+
+            clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID;
+
+            return Promise.resolve(clientConfig);
+        }
+
+        if (!wellknown["m.homeserver"]["base_url"]) {
+            logger.error("No m.homeserver base_url in config");
+
+            clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HS_BASE_URL;
+
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 2: Make sure the homeserver URL is valid *looking*. We'll make
+        // sure it points to a homeserver in Step 3.
+        const hsUrl = this._sanitizeWellKnownUrl(
+            wellknown["m.homeserver"]["base_url"],
+        );
+        if (!hsUrl) {
+            logger.error("Invalid base_url for m.homeserver");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HS_BASE_URL;
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 3: Make sure the homeserver URL points to a homeserver.
+        const hsVersions = await this._fetchWellKnownObject(
+            `${hsUrl}/_matrix/client/versions`,
+        );
+        if (!hsVersions || !hsVersions.raw["versions"]) {
+            logger.error("Invalid /versions response");
+            clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID_HOMESERVER;
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 4: Now that the homeserver looks valid, update our client config.
+        clientConfig["m.homeserver"] = {
+            state: AutoDiscovery.SUCCESS,
+            error: null,
+            base_url: hsUrl,
+        };
+
+        // Step 5: Try to pull out the identity server configuration
+        let isUrl = "";
+        if (wellknown["m.identity_server"]) {
+            // We prepare a failing identity server response to save lines later
+            // in this branch. Note that we also fail the homeserver check in the
+            // object because according to the spec we're supposed to FAIL_ERROR
+            // if *anything* goes wrong with the IS validation, including invalid
+            // format. This means we're supposed to stop discovery completely.
+            const failingClientConfig = {
+                "m.homeserver": {
+                    state: AutoDiscovery.FAIL_ERROR,
+                    error: AutoDiscovery.ERROR_INVALID_IS,
+
+                    // We'll provide the base_url that was previously valid for
+                    // debugging purposes.
+                    base_url: clientConfig["m.homeserver"].base_url,
+                },
+                "m.identity_server": {
+                    state: AutoDiscovery.FAIL_ERROR,
+                    error: AutoDiscovery.ERROR_INVALID_IS,
+                    base_url: null,
+                },
+            };
+
+            // Step 5a: Make sure the URL is valid *looking*. We'll make sure it
+            // points to an identity server in Step 5b.
+            isUrl = this._sanitizeWellKnownUrl(
+                wellknown["m.identity_server"]["base_url"],
+            );
+            if (!isUrl) {
+                logger.error("Invalid base_url for m.identity_server");
+                failingClientConfig["m.identity_server"].error =
+                    AutoDiscovery.ERROR_INVALID_IS_BASE_URL;
+                return Promise.resolve(failingClientConfig);
+            }
+
+            // Step 5b: Verify there is an identity server listening on the provided
+            // URL.
+            const isResponse = await this._fetchWellKnownObject(
+                `${isUrl}/_matrix/identity/api/v1`,
+            );
+            if (!isResponse || !isResponse.raw || isResponse.action !== "SUCCESS") {
+                logger.error("Invalid /api/v1 response");
+                failingClientConfig["m.identity_server"].error =
+                    AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER;
+                return Promise.resolve(failingClientConfig);
+            }
+        }
+
+        // Step 6: Now that the identity server is valid, or never existed,
+        // populate the IS section.
+        if (isUrl && isUrl.length > 0) {
+            clientConfig["m.identity_server"] = {
+                state: AutoDiscovery.SUCCESS,
+                error: null,
+                base_url: isUrl,
+            };
+        }
+
+        // Step 7: Copy any other keys directly into the clientConfig. This is for
+        // things like custom configuration of services.
+        Object.keys(wellknown)
+            .map((k) => {
+                if (k === "m.homeserver" || k === "m.identity_server") {
+                    // Only copy selected parts of the config to avoid overwriting
+                    // properties computed by the validation logic above.
+                    const notProps = ["error", "state", "base_url"];
+                    for (const prop of Object.keys(wellknown[k])) {
+                        if (notProps.includes(prop)) continue;
+                        clientConfig[k][prop] = wellknown[k][prop];
+                    }
+                } else {
+                    // Just copy the whole thing over otherwise
+                    clientConfig[k] = wellknown[k];
+                }
+            });
+
+        // Step 8: Give the config to the caller (finally)
+        return Promise.resolve(clientConfig);
+    }
+
     /**
      * Attempts to automatically discover client configuration information
      * prior to logging in. Such information includes the homeserver URL
@@ -171,7 +381,7 @@ export class AutoDiscovery {
         const clientConfig = {
             "m.homeserver": {
                 state: AutoDiscovery.FAIL_ERROR,
-                error: "Invalid homeserver discovery response",
+                error: AutoDiscovery.ERROR_INVALID,
                 base_url: null,
             },
             "m.identity_server": {
@@ -188,10 +398,8 @@ export class AutoDiscovery {
         const wellknown = await this._fetchWellKnownObject(
             `https://${domain}/.well-known/matrix/client`,
         );
-        if (!wellknown || wellknown.action !== "SUCCESS"
+        if (!wellknown || wellknown.action !== "SUCCESS") {
-            || !wellknown.raw["m.homeserver"]
+            logger.error("No response or error when parsing .well-known");
-            || !wellknown.raw["m.homeserver"]["base_url"]) {
-            logger.error("No m.homeserver key in well-known response");
             if (wellknown.reason) logger.error(wellknown.reason);
             if (wellknown.action === "IGNORE") {
                 clientConfig["m.homeserver"] = {
@@ -202,99 +410,13 @@ export class AutoDiscovery {
             } else {
                 // this can only ever be FAIL_PROMPT at this point.
                 clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+                clientConfig["m.homeserver"].error = AutoDiscovery.ERROR_INVALID;
             }
             return Promise.resolve(clientConfig);
         }
 
-        // Step 2: Make sure the homeserver URL is valid *looking*. We'll make
+        // Step 2: Validate and parse the config
-        // sure it points to a homeserver in Step 3.
+        return AutoDiscovery.fromDiscoveryConfig(wellknown.raw);
-        const hsUrl = this._sanitizeWellKnownUrl(
-            wellknown.raw["m.homeserver"]["base_url"],
-        );
-        if (!hsUrl) {
-            logger.error("Invalid base_url for m.homeserver");
-            return Promise.resolve(clientConfig);
-        }
-
-        // Step 3: Make sure the homeserver URL points to a homeserver.
-        const hsVersions = await this._fetchWellKnownObject(
-            `${hsUrl}/_matrix/client/versions`,
-        );
-        if (!hsVersions || !hsVersions.raw["versions"]) {
-            logger.error("Invalid /versions response");
-            return Promise.resolve(clientConfig);
-        }
-
-        // Step 4: Now that the homeserver looks valid, update our client config.
-        clientConfig["m.homeserver"] = {
-            state: AutoDiscovery.SUCCESS,
-            error: null,
-            base_url: hsUrl,
-        };
-
-        // Step 5: Try to pull out the identity server configuration
-        let isUrl = "";
-        if (wellknown.raw["m.identity_server"]) {
-            // We prepare a failing identity server response to save lines later
-            // in this branch. Note that we also fail the homeserver check in the
-            // object because according to the spec we're supposed to FAIL_ERROR
-            // if *anything* goes wrong with the IS validation, including invalid
-            // format. This means we're supposed to stop discovery completely.
-            const failingClientConfig = {
-                "m.homeserver": {
-                    state: AutoDiscovery.FAIL_ERROR,
-                    error: "Invalid identity server discovery response",
-
-                    // We'll provide the base_url that was previously valid for
-                    // debugging purposes.
-                    base_url: clientConfig["m.homeserver"].base_url,
-                },
-                "m.identity_server": {
-                    state: AutoDiscovery.FAIL_ERROR,
-                    error: "Invalid identity server discovery response",
-                    base_url: null,
-                },
-            };
-
-            // Step 5a: Make sure the URL is valid *looking*. We'll make sure it
-            // points to an identity server in Step 5b.
-            isUrl = this._sanitizeWellKnownUrl(
-                wellknown.raw["m.identity_server"]["base_url"],
-            );
-            if (!isUrl) {
-                logger.error("Invalid base_url for m.identity_server");
-                return Promise.resolve(failingClientConfig);
-            }
-
-            // Step 5b: Verify there is an identity server listening on the provided
-            // URL.
-            const isResponse = await this._fetchWellKnownObject(
-                `${isUrl}/_matrix/identity/api/v1`,
-            );
-            if (!isResponse || !isResponse.raw || isResponse.action !== "SUCCESS") {
-                logger.error("Invalid /api/v1 response");
-                return Promise.resolve(failingClientConfig);
-            }
-        }
-
-        // Step 6: Now that the identity server is valid, or never existed,
-        // populate the IS section.
-        if (isUrl && isUrl.length > 0) {
-            clientConfig["m.identity_server"] = {
-                state: AutoDiscovery.SUCCESS,
-                error: null,
-                base_url: isUrl,
-            };
-        }
-
-        // Step 7: Copy any other keys directly into the clientConfig. This is for
-        // things like custom configuration of services.
-        Object.keys(wellknown.raw)
-            .filter((k) => k !== "m.homeserver" && k !== "m.identity_server")
-            .map((k) => clientConfig[k] = wellknown.raw[k]);
-
-        // Step 8: Give the config to the caller (finally)
-        return Promise.resolve(clientConfig);
     }
 
     /**
@@ -358,14 +480,14 @@ export class AutoDiscovery {
             const request = require("./matrix").getRequest();
             if (!request) throw new Error("No request library available");
             request(
-                { method: "GET", uri: url },
+                { method: "GET", uri: url, timeout: 5000 },
                 (err, response, body) => {
                     if (err || response.statusCode < 200 || response.statusCode >= 300) {
                         let action = "FAIL_PROMPT";
                         let reason = (err ? err.message : null) || "General failure";
                         if (response.statusCode === 404) {
                             action = "IGNORE";
-                            reason = "No .well-known JSON file found";
+                            reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                         }
                         resolve({raw: {}, action: action, reason: reason, error: err});
                         return;
@@ -374,12 +496,15 @@ export class AutoDiscovery {
                     try {
                         resolve({raw: JSON.parse(body), action: "SUCCESS"});
                     } catch (e) {
-                        let reason = "General failure";
+                        let reason = AutoDiscovery.ERROR_INVALID;
-                        if (e.name === "SyntaxError") reason = "Invalid JSON";
+                        if (e.name === "SyntaxError") {
+                            reason = AutoDiscovery.ERROR_INVALID_JSON;
+                        }
                         resolve({
                             raw: {},
                             action: "FAIL_PROMPT",
-                            reason: reason, error: e,
+                            reason: reason,
+                            error: e,
                         });
                     }
                 },

src/base-apis.js

@@ -263,8 +263,7 @@ MatrixBaseApis.prototype.login = function(loginType, data, callback) {
 
     return this._http.authedRequest(
         (error, response) => {
-            if (loginType === "m.login.password" && response &&
+            if (response && response.access_token && response.user_id) {
-                response.access_token && response.user_id) {
                 this._http.opts.accessToken = response.access_token;
                 this.credentials = {
                     userId: response.user_id,
@@ -888,21 +887,6 @@ MatrixBaseApis.prototype.sendStateEvent = function(roomId, eventType, content, s
     );
 };
 
-/**
- * @param {string} roomId
- * @param {string} eventId
- * @param {module:client.callback} callback Optional.
- * @return {module:client.Promise} Resolves: TODO
- * @return {module:http-api.MatrixError} Rejects: with an error response.
- */
-MatrixBaseApis.prototype.redactEvent = function(roomId, eventId, callback) {
-    const path = utils.encodeUri("/rooms/$roomId/redact/$eventId", {
-        $roomId: roomId,
-        $eventId: eventId,
-    });
-    return this._http.authedRequest(callback, "POST", path, undefined, {});
-};
-
 /**
  * @param {string} roomId
  * @param {Number} limit

src/client.js

@@ -45,6 +45,7 @@ const olmlib = require("./crypto/olmlib");
 
 import ReEmitter from './ReEmitter';
 import RoomList from './crypto/RoomList';
+import logger from '../src/logger';
 
 import Crypto from './crypto';
 import { isCryptoAvailable } from './crypto';
@@ -73,7 +74,7 @@ function keysFromRecoverySession(sessions, decryptionKey, roomId) {
             decrypted.room_id = roomId;
             keys.push(decrypted);
         } catch (e) {
-            console.log("Failed to decrypt session from backup");
+            logger.log("Failed to decrypt session from backup");
         }
     }
     return keys;
@@ -151,6 +152,11 @@ function keyFromRecoverySession(session, decryptionKey) {
  * maintain support for back-paginating the live timeline after a '/sync'
  * result with a gap.
  *
+ * @param {boolean} [opts.unstableClientRelationAggregation = false]
+ * Optional. Set to true to enable client-side aggregation of event relations
+ * via `EventTimelineSet#getRelationsForEvent`.
+ * This feature is currently unstable and the API may change without notice.
+ *
  * @param {Array} [opts.verificationMethods] Optional. The verification method
  * that the application can handle.  Each element should be an item from {@link
  * module:crypto~verificationMethods verificationMethods}, or a class that
@@ -216,6 +222,7 @@ function MatrixClient(opts) {
     this.timelineSupport = Boolean(opts.timelineSupport);
     this.urlPreviewCache = {};
     this._notifTimelineSet = null;
+    this.unstableClientRelationAggregation = !!opts.unstableClientRelationAggregation;
 
     this._crypto = null;
     this._cryptoStore = opts.cryptoStore;
@@ -245,19 +252,32 @@ function MatrixClient(opts) {
         const actions = this._pushProcessor.actionsForEvent(event);
         event.setPushActions(actions); // Might as well while we're here
 
+        const room = this.getRoom(event.getRoomId());
+        if (!room) return;
+
+        const currentCount = room.getUnreadNotificationCount("highlight");
+
         // Ensure the unread counts are kept up to date if the event is encrypted
+        // We also want to make sure that the notification count goes up if we already
+        // have encrypted events to avoid other code from resetting 'highlight' to zero.
         const oldHighlight = oldActions && oldActions.tweaks
             ? !!oldActions.tweaks.highlight : false;
         const newHighlight = actions && actions.tweaks
             ? !!actions.tweaks.highlight : false;
-        if (oldHighlight !== newHighlight) {
+        if (oldHighlight !== newHighlight || currentCount > 0) {
-            const room = this.getRoom(event.getRoomId());
             // TODO: Handle mentions received while the client is offline
             // See also https://github.com/vector-im/riot-web/issues/9069
-            if (room && !room.hasUserReadEvent(this.getUserId(), event.getId())) {
+            if (!room.hasUserReadEvent(this.getUserId(), event.getId())) {
-                const current = room.getUnreadNotificationCount("highlight");
+                let newCount = currentCount;
-                const newCount = newHighlight ? current + 1 : current - 1;
+                if (newHighlight && !oldHighlight) newCount++;
+                if (!newHighlight && oldHighlight) newCount--;
                 room.setUnreadNotificationCount("highlight", newCount);
+
+                // Fix 'Mentions Only' rooms from not having the right badge count
+                const totalCount = room.getUnreadNotificationCount('total');
+                if (totalCount < newCount) {
+                    room.setUnreadNotificationCount('total', newCount);
+                }
             }
         }
     });
@@ -431,14 +451,16 @@ MatrixClient.prototype.setNotifTimelineSet = function(notifTimelineSet) {
 /**
  * Gets the capabilities of the homeserver. Always returns an object of
  * capability keys and their options, which may be empty.
+ * @param {boolean} fresh True to ignore any cached values.
  * @return {module:client.Promise} Resolves to the capabilities of the homeserver
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
-MatrixClient.prototype.getCapabilities = function() {
+MatrixClient.prototype.getCapabilities = function(fresh=false) {
-    if (this._cachedCapabilities) {
     const now = new Date().getTime();
-        if (now - this._cachedCapabilities.lastUpdated <= CAPABILITIES_CACHE_MS) {
+
-            console.log("Returning cached capabilities");
+    if (this._cachedCapabilities && !fresh) {
+        if (now < this._cachedCapabilities.expiration) {
+            logger.log("Returning cached capabilities");
             return Promise.resolve(this._cachedCapabilities.capabilities);
         }
     }
@@ -446,15 +468,25 @@ MatrixClient.prototype.getCapabilities = function() {
     // We swallow errors because we need a default object anyhow
     return this._http.authedRequest(
         undefined, "GET", "/capabilities",
-    ).catch(() => null).then((r) => {
+    ).catch((e) => {
+        logger.error(e);
+        return null; // otherwise consume the error
+    }).then((r) => {
         if (!r) r = {};
         const capabilities = r["capabilities"] || {};
+
+        // If the capabilities missed the cache, cache it for a shorter amount
+        // of time to try and refresh them later.
+        const cacheMs = Object.keys(capabilities).length
+            ? CAPABILITIES_CACHE_MS
+            : 60000 + (Math.random() * 5000);
+
         this._cachedCapabilities = {
             capabilities: capabilities,
-            lastUpdated: new Date().getTime(),
+            expiration: now + cacheMs,
         };
 
-        console.log("Caching capabilities: ", capabilities);
+        logger.log("Caching capabilities: ", capabilities);
         return capabilities;
     });
 };
@@ -480,7 +512,7 @@ MatrixClient.prototype.initCrypto = async function() {
     }
 
     if (this._crypto) {
-        console.warn("Attempt to re-initialise e2e encryption on MatrixClient");
+        logger.warn("Attempt to re-initialise e2e encryption on MatrixClient");
         return;
     }
 
@@ -494,7 +526,7 @@ MatrixClient.prototype.initCrypto = async function() {
     }
 
     // initialise the list of encrypted rooms (whether or not crypto is enabled)
-    console.log("Crypto: initialising roomlist...");
+    logger.log("Crypto: initialising roomlist...");
     await this._roomList.init();
 
     const userId = this.getUserId();
@@ -529,7 +561,7 @@ MatrixClient.prototype.initCrypto = async function() {
         "crypto.warning",
     ]);
 
-    console.log("Crypto: initialising crypto object...");
+    logger.log("Crypto: initialising crypto object...");
     await crypto.init();
 
     this.olmVersion = Crypto.getOlmVersion();
@@ -1407,7 +1439,7 @@ MatrixClient.prototype._restoreKeyBackup = async function(
                 key.session_id = targetSessionId;
                 keys.push(key);
             } catch (e) {
-                console.log("Failed to decrypt session from backup");
+                logger.log("Failed to decrypt session from backup");
             }
         }
 
@@ -1851,6 +1883,21 @@ MatrixClient.prototype.setPowerLevel = function(roomId, userId, powerLevel,
  */
 MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
                                             callback) {
+    return this._sendCompleteEvent(roomId, {
+        type: eventType,
+        content: content,
+    }, txnId, callback);
+};
+/**
+ * @param {string} roomId
+ * @param {object} eventObject An object with the partial structure of an event, to which event_id, user_id, room_id and origin_server_ts will be added.
+ * @param {string} txnId the txnId.
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: TODO
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
+                                            callback) {
     if (utils.isFunction(txnId)) {
         callback = txnId; txnId = undefined;
     }
@@ -1859,22 +1906,22 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
         txnId = this.makeTxnId();
     }
 
-    console.log(`sendEvent of type ${eventType} in ${roomId} with txnId ${txnId}`);
+    const localEvent = new MatrixEvent(Object.assign(eventObject, {
+        event_id: "~" + roomId + ":" + txnId,
+        user_id: this.credentials.userId,
+        room_id: roomId,
+        origin_server_ts: new Date().getTime(),
+    }));
+
+    const type = localEvent.getType();
+    logger.log(`sendEvent of type ${type} in ${roomId} with txnId ${txnId}`);
 
     // we always construct a MatrixEvent when sending because the store and
     // scheduler use them. We'll extract the params back out if it turns out
     // the client has no scheduler or store.
     const room = this.getRoom(roomId);
-    const localEvent = new MatrixEvent({
-        event_id: "~" + roomId + ":" + txnId,
-        user_id: this.credentials.userId,
-        room_id: roomId,
-        type: eventType,
-        origin_server_ts: new Date().getTime(),
-        content: content,
-    });
     localEvent._txnId = txnId;
-    localEvent.status = EventStatus.SENDING;
+    localEvent.setStatus(EventStatus.SENDING);
 
     // add this event immediately to the local store as 'sending'.
     if (room) {
@@ -1941,7 +1988,7 @@ function _sendEvent(client, room, event, callback) {
         return res;
     }, function(err) {
         // the request failed to send.
-        console.error("Error sending event", err.stack || err);
+        logger.error("Error sending event", err.stack || err);
 
         try {
             // set the error on the event before we update the status:
@@ -1957,7 +2004,7 @@ function _sendEvent(client, room, event, callback) {
                 callback(err);
             }
         } catch (err2) {
-            console.error("Exception in error handler!", err2.stack || err);
+            logger.error("Exception in error handler!", err2.stack || err);
         }
         throw err;
     });
@@ -2004,7 +2051,7 @@ function _updatePendingEventStatus(room, event, newStatus) {
     if (room) {
         room.updatePendingEvent(event, newStatus);
     } else {
-        event.status = newStatus;
+        event.setStatus(newStatus);
     }
 }
 
@@ -2026,6 +2073,9 @@ function _sendEventHttpRequest(client, event) {
             pathTemplate = "/rooms/$roomId/state/$eventType/$stateKey";
         }
         path = utils.encodeUri(pathTemplate, pathParams);
+    } else if (event.getType() === "m.room.redaction") {
+        const pathTemplate = `/rooms/$roomId/redact/${event.event.redacts}/$txnId`;
+        path = utils.encodeUri(pathTemplate, pathParams);
     } else {
         path = utils.encodeUri(
             "/rooms/$roomId/send/$eventType/$txnId", pathParams,
@@ -2035,13 +2085,30 @@ function _sendEventHttpRequest(client, event) {
     return client._http.authedRequest(
         undefined, "PUT", path, undefined, event.getWireContent(),
     ).then((res) => {
-        console.log(
+        logger.log(
             `Event sent to ${event.getRoomId()} with event id ${res.event_id}`,
         );
         return res;
     });
 }
 
+/**
+ * @param {string} roomId
+ * @param {string} eventId
+ * @param {string} [txnId]  transaction id. One will be made up if not
+ *    supplied.
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: TODO
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.redactEvent = function(roomId, eventId, txnId, callback) {
+    return this._sendCompleteEvent(roomId, {
+        type: "m.room.redaction",
+        content: {},
+        redacts: eventId,
+    }, txnId, callback);
+};
+
 /**
  * @param {string} roomId
  * @param {Object} content
@@ -2342,10 +2409,10 @@ MatrixClient.prototype.getRoomUpgradeHistory = function(roomId, verifyLinks=fals
     // Work backwards first, looking at create events.
     let createEvent = currentRoom.currentState.getStateEvents("m.room.create", "");
     while (createEvent) {
-        console.log(`Looking at ${createEvent.getId()}`);
+        logger.log(`Looking at ${createEvent.getId()}`);
         const predecessor = createEvent.getContent()['predecessor'];
         if (predecessor && predecessor['room_id']) {
-            console.log(`Looking at predecessor ${predecessor['room_id']}`);
+            logger.log(`Looking at predecessor ${predecessor['room_id']}`);
             const refRoom = this.getRoom(predecessor['room_id']);
             if (!refRoom) break; // end of the chain
 
@@ -3710,7 +3777,7 @@ MatrixClient.prototype.syncLeftRooms = function() {
 
     // cleanup locks
     this._syncLeftRoomsPromise.then(function(res) {
-        console.log("Marking success of sync left room request");
+        logger.log("Marking success of sync left room request");
         self._syncedLeftRooms = true; // flip the bit on success
     }).finally(function() {
         self._syncLeftRoomsPromise = null; // cleanup ongoing request state
@@ -3955,7 +4022,7 @@ MatrixClient.prototype.startClient = async function(opts) {
 
     if (this._syncApi) {
         // This shouldn't happen since we thought the client was not running
-        console.error("Still have sync object whilst not running: stopping old one");
+        logger.error("Still have sync object whilst not running: stopping old one");
         this._syncApi.stop();
     }
 
@@ -3998,7 +4065,7 @@ MatrixClient.prototype._storeClientOptions = function() {
  * clean shutdown.
  */
 MatrixClient.prototype.stopClient = function() {
-    console.log('stopping MatrixClient');
+    logger.log('stopping MatrixClient');
 
     this.clientRunning = false;
     // TODO: f.e. Room => self.store.storeRoom(room) ?
@@ -4139,7 +4206,7 @@ function setupCallEventHandler(client) {
                 return; // stale/old invite event
             }
             if (call) {
-                console.log(
+                logger.log(
                     "WARN: Already have a MatrixCall with id %s but got an " +
                     "invite. Clobbering.",
                     content.call_id,
@@ -4150,7 +4217,7 @@ function setupCallEventHandler(client) {
                 forceTURN: client._forceTURN,
             });
             if (!call) {
-                console.log(
+                logger.log(
                     "Incoming call ID " + content.call_id + " but this client " +
                     "doesn't support WebRTC",
                 );
@@ -4195,14 +4262,14 @@ function setupCallEventHandler(client) {
                 if (existingCall.state === 'wait_local_media' ||
                         existingCall.state === 'create_offer' ||
                         existingCall.callId > call.callId) {
-                    console.log(
+                    logger.log(
                         "Glare detected: answering incoming call " + call.callId +
                         " and canceling outgoing call " + existingCall.callId,
                     );
                     existingCall._replacedBy(call);
                     call.answer();
                 } else {
-                    console.log(
+                    logger.log(
                         "Glare detected: rejecting incoming call " + call.callId +
                         " and keeping outgoing call " + existingCall.callId,
                     );
@@ -4272,7 +4339,7 @@ function checkTurnServers(client) {
 
     client.turnServer().done(function(res) {
         if (res.uris) {
-            console.log("Got TURN URIs: " + res.uris + " refresh in " +
+            logger.log("Got TURN URIs: " + res.uris + " refresh in " +
                 res.ttl + " secs");
             // map the response to a format that can be fed to
             // RTCPeerConnection
@@ -4288,7 +4355,7 @@ function checkTurnServers(client) {
             }, (res.ttl || (60 * 60)) * 1000 * 0.9);
         }
     }, function(err) {
-        console.error("Failed to get TURN URIs");
+        logger.error("Failed to get TURN URIs");
         client._checkTurnServersTimeoutID =
             setTimeout(function() {
  checkTurnServers(client);
@@ -4319,6 +4386,10 @@ function _PojoToMatrixEventMapper(client) {
             ]);
             event.attemptDecryption(client._crypto);
         }
+        const room = client.getRoom(event.getRoomId());
+        if (room) {
+            room.reEmitter.reEmit(event, ["Event.replaced"]);
+        }
         return event;
     }
     return mapper;

src/content-repo.js

@@ -51,10 +51,10 @@ module.exports = {
         const params = {};
 
         if (width) {
-            params.width = width;
+            params.width = Math.round(width);
         }
         if (height) {
-            params.height = height;
+            params.height = Math.round(height);
         }
         if (resizeMethod) {
             params.method = resizeMethod;

src/crypto/OlmDevice.js

@@ -462,7 +462,7 @@ OlmDevice.prototype.createInboundSession = async function(
  */
 OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityKey) {
     if (this._sessionsInProgress[theirDeviceIdentityKey]) {
-        console.log("waiting for session to be created");
+        logger.log("waiting for session to be created");
         try {
             await this._sessionsInProgress[theirDeviceIdentityKey];
         } catch (e) {

src/crypto/algorithms/megolm.js

@@ -23,8 +23,8 @@ limitations under the License.
  */
 
 import Promise from 'bluebird';
+import logger from '../../../src/logger';
 
-const logger = require("../../logger");
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
 const base = require("./base");
@@ -278,7 +278,7 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
         ).catch((e) => {
             // This throws if the upload failed, but this is fine
             // since it will have written it to the db and will retry.
-            console.log("Failed to back up group session", e);
+            logger.log("Failed to back up group session", e);
         });
     }
 
@@ -955,7 +955,7 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
             ).catch((e) => {
                 // This throws if the upload failed, but this is fine
                 // since it will have written it to the db and will retry.
-                console.log("Failed to back up group session", e);
+                logger.log("Failed to back up group session", e);
             });
         }
     }).catch((e) => {
@@ -1088,7 +1088,7 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
             ).catch((e) => {
                 // This throws if the upload failed, but this is fine
                 // since it will have written it to the db and will retry.
-                console.log("Failed to back up group session", e);
+                logger.log("Failed to back up group session", e);
             });
         }
         // have another go at decrypting events sent with this session.

src/crypto/algorithms/olm.js

@@ -22,7 +22,7 @@ limitations under the License.
  */
 import Promise from 'bluebird';
 
-const logger = require("../../logger");
+import logger from '../../logger';
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
 const DeviceInfo = require("../deviceinfo");

src/crypto/index.js

@@ -26,7 +26,7 @@ const anotherjson = require('another-json');
 import Promise from 'bluebird';
 import {EventEmitter} from 'events';
 
-const logger = require("../logger");
+import logger from '../logger';
 const utils = require("../utils");
 const OlmDevice = require("./OlmDevice");
 const olmlib = require("./olmlib");
@@ -218,11 +218,11 @@ utils.inherits(Crypto, EventEmitter);
  * Returns a promise which resolves once the crypto module is ready for use.
  */
 Crypto.prototype.init = async function() {
-    console.log("Crypto: initialising Olm...");
+    logger.log("Crypto: initialising Olm...");
     await global.Olm.init();
-    console.log("Crypto: initialising Olm device...");
+    logger.log("Crypto: initialising Olm device...");
     await this._olmDevice.init();
-    console.log("Crypto: loading device list...");
+    logger.log("Crypto: loading device list...");
     await this._deviceList.load();
 
     // build our device keys: these will later be uploaded
@@ -231,7 +231,7 @@ Crypto.prototype.init = async function() {
     this._deviceKeys["curve25519:" + this._deviceId] =
         this._olmDevice.deviceCurve25519Key;
 
-    console.log("Crypto: fetching own devices...");
+    logger.log("Crypto: fetching own devices...");
     let myDevices = this._deviceList.getRawStoredDevicesForUser(
         this._userId,
     );
@@ -242,7 +242,7 @@ Crypto.prototype.init = async function() {
 
     if (!myDevices[this._deviceId]) {
         // add our own deviceinfo to the cryptoStore
-        console.log("Crypto: adding this device to the store...");
+        logger.log("Crypto: adding this device to the store...");
         const deviceInfo = {
             keys: this._deviceKeys,
             algorithms: this._supportedAlgorithms,
@@ -260,7 +260,7 @@ Crypto.prototype.init = async function() {
     // (this is important for key backups & things)
     this._deviceList.startTrackingDeviceList(this._userId);
 
-    console.log("Crypto: checking for key backup...");
+    logger.log("Crypto: checking for key backup...");
     this._checkAndStartKeyBackup();
 };
 
@@ -479,9 +479,9 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
  * to it.
  */
 Crypto.prototype._checkAndStartKeyBackup = async function() {
-    console.log("Checking key backup status...");
+    logger.log("Checking key backup status...");
     if (this._baseApis.isGuest()) {
-        console.log("Skipping key backup check since user is guest");
+        logger.log("Skipping key backup check since user is guest");
         this._checkedForBackup = true;
         return null;
     }
@@ -489,7 +489,7 @@ Crypto.prototype._checkAndStartKeyBackup = async function() {
     try {
         backupInfo = await this._baseApis.getKeyBackupVersion();
     } catch (e) {
-        console.log("Error checking for active key backup", e);
+        logger.log("Error checking for active key backup", e);
         if (e.httpStatus / 100 === 4) {
             // well that's told us. we won't try again.
             this._checkedForBackup = true;
@@ -501,27 +501,27 @@ Crypto.prototype._checkAndStartKeyBackup = async function() {
     const trustInfo = await this.isKeyBackupTrusted(backupInfo);
 
     if (trustInfo.usable && !this.backupInfo) {
-        console.log(
+        logger.log(
             "Found usable key backup v" + backupInfo.version +
             ": enabling key backups",
         );
         this._baseApis.enableKeyBackup(backupInfo);
     } else if (!trustInfo.usable && this.backupInfo) {
-        console.log("No usable key backup: disabling key backup");
+        logger.log("No usable key backup: disabling key backup");
         this._baseApis.disableKeyBackup();
     } else if (!trustInfo.usable && !this.backupInfo) {
-        console.log("No usable key backup: not enabling key backup");
+        logger.log("No usable key backup: not enabling key backup");
     } else if (trustInfo.usable && this.backupInfo) {
         // may not be the same version: if not, we should switch
         if (backupInfo.version !== this.backupInfo.version) {
-            console.log(
+            logger.log(
                 "On backup version " + this.backupInfo.version + " but found " +
                 "version " + backupInfo.version + ": switching.",
             );
             this._baseApis.disableKeyBackup();
             this._baseApis.enableKeyBackup(backupInfo);
         } else {
-            console.log("Backup version " + backupInfo.version + " still current");
+            logger.log("Backup version " + backupInfo.version + " still current");
         }
     }
 
@@ -591,7 +591,7 @@ Crypto.prototype.isKeyBackupTrusted = async function(backupInfo) {
     for (const keyId of Object.keys(mySigs)) {
         const keyIdParts = keyId.split(':');
         if (keyIdParts[0] !== 'ed25519') {
-            console.log("Ignoring unknown signature type: " + keyIdParts[0]);
+            logger.log("Ignoring unknown signature type: " + keyIdParts[0]);
             continue;
         }
         // Could be an SSK but just say this is the device ID for backwards compat
@@ -636,7 +636,11 @@ Crypto.prototype.isKeyBackupTrusted = async function(backupInfo) {
                 );
                 sigInfo.valid = true;
             } catch (e) {
-                logger.info("Bad signature from key ID " + keyId, e);
+                logger.info(
+                    "Bad signature from key ID " + keyId + " userID " + this._userId +
+                    " device ID " + device.deviceId + " fingerprint: " +
+                    device.getFingerprint(), backupInfo.auth_data, e,
+                );
                 sigInfo.valid = false;
             }
         } else {
@@ -1528,7 +1532,7 @@ Crypto.prototype.scheduleKeyBackupSend = async function(maxDelay = 10000) {
                 numFailures = 0;
             } catch (err) {
                 numFailures++;
-                console.log("Key backup request failed", err);
+                logger.log("Key backup request failed", err);
                 if (err.data) {
                     if (
                         err.data.errcode == 'M_NOT_FOUND' ||
@@ -2240,7 +2244,7 @@ Crypto.prototype._onKeyVerificationMessage = function(event) {
     if (!handler) {
         return;
     } else if (event.getType() === "m.key.verification.cancel") {
-        console.log(event);
+        logger.log(event);
         if (handler.verifier) {
             handler.verifier.cancel(event);
         } else if (handler.request && handler.request.cancel) {

src/crypto/olmlib.js

@@ -24,7 +24,7 @@ limitations under the License.
 import Promise from 'bluebird';
 const anotherjson = require('another-json');
 
-const logger = require("../logger");
+import logger from '../logger';
 const utils = require("../utils");
 
 /**

src/crypto/recoverykey.js

@@ -21,7 +21,7 @@ import bs58 from 'bs58';
 const OLM_RECOVERY_KEY_PREFIX = [0x8B, 0x01];
 
 export function encodeRecoveryKey(key) {
-    const buf = new Uint8Array(OLM_RECOVERY_KEY_PREFIX.length + key.length + 1);
+    const buf = new Buffer(OLM_RECOVERY_KEY_PREFIX.length + key.length + 1);
     buf.set(OLM_RECOVERY_KEY_PREFIX, 0);
     buf.set(key, OLM_RECOVERY_KEY_PREFIX.length);
 

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -711,7 +711,7 @@ function promiseifyTxn(txn) {
             if (txn._mx_abortexception !== undefined) {
                 reject(txn._mx_abortexception);
             } else {
-                console.log("Error performing indexeddb txn", event);
+                logger.log("Error performing indexeddb txn", event);
                 reject(event.target.error);
             }
         };
@@ -719,7 +719,7 @@ function promiseifyTxn(txn) {
             if (txn._mx_abortexception !== undefined) {
                 reject(txn._mx_abortexception);
             } else {
-                console.log("Error performing indexeddb txn", event);
+                localStorage.log("Error performing indexeddb txn", event);
                 reject(event.target.error);
             }
         };

src/crypto/store/indexeddb-crypto-store.js

@@ -90,7 +90,7 @@ export default class IndexedDBCryptoStore {
             };
 
             req.onerror = (ev) => {
-                console.log("Error connecting to indexeddb", ev);
+                logger.log("Error connecting to indexeddb", ev);
                 reject(ev.target.error);
             };
 
@@ -160,7 +160,7 @@ export default class IndexedDBCryptoStore {
             };
 
             req.onerror = (ev) => {
-                console.log("Error deleting data from indexeddb", ev);
+                logger.log("Error deleting data from indexeddb", ev);
                 reject(ev.target.error);
             };
 

src/crypto/verification/Base.js

@@ -183,17 +183,29 @@ export default class VerificationBase extends EventEmitter {
     }
 
     async _verifyKeys(userId, keys, verifier) {
+        // we try to verify all the keys that we're told about, but we might
+        // not know about all of them, so keep track of the keys that we know
+        // about, and ignore the rest
+        const verifiedDevices = [];
+
         for (const [keyId, keyInfo] of Object.entries(keys)) {
             const deviceId = keyId.split(':', 2)[1];
             const device = await this._baseApis.getStoredDevice(userId, deviceId);
             if (!device) {
-                throw new Error(`Could not find device ${deviceId}`);
+                logger.warn(`verification: Could not find device ${deviceId} to verify`);
             } else {
                 await verifier(keyId, device, keyInfo);
+                verifiedDevices.push(deviceId);
             }
         }
-        for (const keyId of Object.keys(keys)) {
+
-            const deviceId = keyId.split(':', 2)[1];
+        // if none of the keys could be verified, then error because the app
+        // should be informed about that
+        if (!verifiedDevices.length) {
+            throw new Error("No devices could be verified");
+        }
+
+        for (const deviceId of verifiedDevices) {
             await this._baseApis.setDeviceVerified(userId, deviceId);
         }
     }

src/crypto/verification/SAS.js

@@ -143,17 +143,44 @@ function generateEmojiSas(sasBytes) {
     return emojis.map((num) => emojiMapping[num]);
 }
 
+const sasGenerators = {
+    decimal: generateDecimalSas,
+    emoji: generateEmojiSas,
+};
+
 function generateSas(sasBytes, methods) {
     const sas = {};
-    if (methods.includes("decimal")) {
+    for (const method of methods) {
-        sas["decimal"] = generateDecimalSas(sasBytes);
+        if (method in sasGenerators) {
+            sas[method] = sasGenerators[method](sasBytes);
         }
-    if (methods.includes("emoji")) {
-        sas["emoji"] = generateEmojiSas(sasBytes);
     }
     return sas;
 }
 
+const macMethods = {
+    "hkdf-hmac-sha256": "calculate_mac",
+    "hmac-sha256": "calculate_mac_long_kdf",
+};
+
+/* lists of algorithms/methods that are supported.  The key agreement, hashes,
+ * and MAC lists should be sorted in order of preference (most preferred
+ * first).
+ */
+const KEY_AGREEMENT_LIST = ["curve25519"];
+const HASHES_LIST = ["sha256"];
+const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
+const SAS_LIST = Object.keys(sasGenerators);
+
+const KEY_AGREEMENT_SET = new Set(KEY_AGREEMENT_LIST);
+const HASHES_SET = new Set(HASHES_LIST);
+const MAC_SET = new Set(MAC_LIST);
+const SAS_SET = new Set(SAS_LIST);
+
+function intersection(anArray, aSet) {
+    return anArray instanceof Array ? anArray.filter(x => aSet.has(x)) : [];
+}
+
 /**
  * @alias module:crypto/verification/SAS
  * @extends {module:crypto/verification/Base}
@@ -181,11 +208,11 @@ export default class SAS extends Base {
         const initialMessage = {
             method: SAS.NAME,
             from_device: this._baseApis.deviceId,
-            key_agreement_protocols: ["curve25519"],
+            key_agreement_protocols: KEY_AGREEMENT_LIST,
-            hashes: ["sha256"],
+            hashes: HASHES_LIST,
-            message_authentication_codes: ["hmac-sha256"],
+            message_authentication_codes: MAC_LIST,
             // FIXME: allow app to specify what SAS methods can be used
-            short_authentication_string: ["decimal", "emoji"],
+            short_authentication_string: SAS_LIST,
             transaction_id: this.transactionId,
         };
         this._sendToDevice("m.key.verification.start", initialMessage);
@@ -193,19 +220,19 @@ export default class SAS extends Base {
 
         let e = await this._waitForEvent("m.key.verification.accept");
         let content = e.getContent();
-        if (!(content.key_agreement_protocol === "curve25519"
+        const sasMethods
-              && content.hash === "sha256"
+              = intersection(content.short_authentication_string, SAS_SET);
-              && content.message_authentication_code === "hmac-sha256"
+        if (!(KEY_AGREEMENT_SET.has(content.key_agreement_protocol)
-              && content.short_authentication_string instanceof Array
+              && HASHES_SET.has(content.hash)
-              && (content.short_authentication_string.includes("decimal")
+              && MAC_SET.has(content.message_authentication_code)
-                  || content.short_authentication_string.includes("emoji")))) {
+              && sasMethods.length)) {
             throw newUnknownMethodError();
         }
         if (typeof content.commitment !== "string") {
             throw newInvalidMessageError();
         }
+        const macMethod = content.message_authentication_code;
         const hashCommitment = content.commitment;
-        const sasMethods = content.short_authentication_string;
         const olmSAS = new global.Olm.SAS();
         try {
             this._sendToDevice("m.key.verification.key", {
@@ -217,6 +244,7 @@ export default class SAS extends Base {
             // FIXME: make sure event is properly formed
             content = e.getContent();
             const commitmentStr = content.key + anotherjson.stringify(initialMessage);
+            // TODO: use selected hash function (when we support multiple)
             if (olmutil.sha256(commitmentStr) !== hashCommitment) {
                 throw newMismatchedCommitmentError();
             }
@@ -231,7 +259,7 @@ export default class SAS extends Base {
                 this.emit("show_sas", {
                     sas: generateSas(sasBytes, sasMethods),
                     confirm: () => {
-                        this._sendMAC(olmSAS);
+                        this._sendMAC(olmSAS, macMethod);
                         resolve();
                     },
                     cancel: () => reject(newUserCancelledError()),
@@ -245,7 +273,7 @@ export default class SAS extends Base {
                 verifySAS,
             ]);
             content = e.getContent();
-            await this._checkMAC(olmSAS, content);
+            await this._checkMAC(olmSAS, content, macMethod);
         } finally {
             olmSAS.free();
         }
@@ -253,34 +281,37 @@ export default class SAS extends Base {
 
     async _doRespondVerification() {
         let content = this.startEvent.getContent();
-        if (!(content.key_agreement_protocols instanceof Array
+        // Note: we intersect using our pre-made lists, rather than the sets,
-              && content.key_agreement_protocols.includes("curve25519")
+        // so that the result will be in our order of preference.  Then
-              && content.hashes instanceof Array
+        // fetching the first element from the array will give our preferred
-              && content.hashes.includes("sha256")
+        // method out of the ones offered by the other party.
-              && content.message_authentication_codes instanceof Array
+        const keyAgreement
-              && content.message_authentication_codes.includes("hmac-sha256")
+              = intersection(
-              && content.short_authentication_string instanceof Array
+                  KEY_AGREEMENT_LIST, new Set(content.key_agreement_protocols),
-              && (content.short_authentication_string.includes("decimal")
+              )[0];
-                  || content.short_authentication_string.includes("emoji")))) {
+        const hashMethod
+              = intersection(HASHES_LIST, new Set(content.hashes))[0];
+        const macMethod
+              = intersection(MAC_LIST, new Set(content.message_authentication_codes))[0];
+        // FIXME: allow app to specify what SAS methods can be used
+        const sasMethods
+              = intersection(content.short_authentication_string, SAS_SET);
+        if (!(keyAgreement !== undefined
+              && hashMethod !== undefined
+              && macMethod !== undefined
+              && sasMethods.length)) {
             throw newUnknownMethodError();
         }
 
         const olmSAS = new global.Olm.SAS();
-        const sasMethods = [];
-        // FIXME: allow app to specify what SAS methods can be used
-        if (content.short_authentication_string.includes("decimal")) {
-            sasMethods.push("decimal");
-        }
-        if (content.short_authentication_string.includes("emoji")) {
-            sasMethods.push("emoji");
-        }
         try {
             const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
             this._sendToDevice("m.key.verification.accept", {
-                key_agreement_protocol: "curve25519",
+                key_agreement_protocol: keyAgreement,
-                hash: "sha256",
+                hash: hashMethod,
-                message_authentication_code: "hmac-sha256",
+                message_authentication_code: macMethod,
                 short_authentication_string: sasMethods,
+                // TODO: use selected hash function (when we support multiple)
                 commitment: olmutil.sha256(commitmentStr),
             });
 
@@ -302,7 +333,7 @@ export default class SAS extends Base {
                 this.emit("show_sas", {
                     sas: generateSas(sasBytes, sasMethods),
                     confirm: () => {
-                        this._sendMAC(olmSAS);
+                        this._sendMAC(olmSAS, macMethod);
                         resolve();
                     },
                     cancel: () => reject(newUserCancelledError()),
@@ -316,13 +347,13 @@ export default class SAS extends Base {
                 verifySAS,
             ]);
             content = e.getContent();
-            await this._checkMAC(olmSAS, content);
+            await this._checkMAC(olmSAS, content, macMethod);
         } finally {
             olmSAS.free();
         }
     }
 
-    _sendMAC(olmSAS) {
+    _sendMAC(olmSAS, method) {
         const keyId = `ed25519:${this._baseApis.deviceId}`;
         const mac = {};
         const baseInfo = "MATRIX_KEY_VERIFICATION_MAC"
@@ -330,24 +361,24 @@ export default class SAS extends Base {
               + this.userId + this.deviceId
               + this.transactionId;
 
-        mac[keyId] = olmSAS.calculate_mac(
+        mac[keyId] = olmSAS[macMethods[method]](
             this._baseApis.getDeviceEd25519Key(),
             baseInfo + keyId,
         );
-        const keys = olmSAS.calculate_mac(
+        const keys = olmSAS[macMethods[method]](
             keyId,
             baseInfo + "KEY_IDS",
         );
         this._sendToDevice("m.key.verification.mac", { mac, keys });
     }
 
-    async _checkMAC(olmSAS, content) {
+    async _checkMAC(olmSAS, content, method) {
         const baseInfo = "MATRIX_KEY_VERIFICATION_MAC"
               + this.userId + this.deviceId
               + this._baseApis.getUserId() + this._baseApis.deviceId
               + this.transactionId;
 
-        if (content.keys !== olmSAS.calculate_mac(
+        if (content.keys !== olmSAS[macMethods[method]](
             Object.keys(content.mac).sort().join(","),
             baseInfo + "KEY_IDS",
         )) {
@@ -355,7 +386,7 @@ export default class SAS extends Base {
         }
 
         await this._verifyKeys(this.userId, content.mac, (keyId, device, keyInfo) => {
-            if (keyInfo !== olmSAS.calculate_mac(
+            if (keyInfo !== olmSAS[macMethods[method]](
                 device.keys[keyId],
                 baseInfo + keyId,
             )) {

src/http-api.js

@@ -22,6 +22,7 @@ import Promise from 'bluebird';
 const parseContentType = require('content-type').parse;
 
 const utils = require("./utils");
+import logger from '../src/logger';
 
 // we use our own implementation of setTimeout, so that if we get suspended in
 // the middle of a /sync, we cancel the sync as soon as we awake, rather than
@@ -175,7 +176,7 @@ module.exports.MatrixHttpApi.prototype = {
             if (global.XMLHttpRequest) {
                 rawResponse = false;
             } else {
-                console.warn(
+                logger.warn(
                     "Returning the raw JSON from uploadContent(). Future " +
                     "versions of the js-sdk will change this default, to " +
                     "return the parsed object. Set opts.rawResponse=false " +
@@ -188,7 +189,7 @@ module.exports.MatrixHttpApi.prototype = {
         let onlyContentUri = opts.onlyContentUri;
         if (!rawResponse && onlyContentUri === undefined) {
             if (global.XMLHttpRequest) {
-                console.warn(
+                logger.warn(
                     "Returning only the content-uri from uploadContent(). " +
                     "Future versions of the js-sdk will change this " +
                     "default, to return the whole response object. Set " +

src/interactive-auth.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +22,7 @@ import Promise from 'bluebird';
 const url = require("url");
 
 const utils = require("./utils");
+import logger from '../src/logger';
 
 const EMAIL_STAGE_TYPE = "m.login.email.identity";
 const MSISDN_STAGE_TYPE = "m.login.msisdn";
@@ -88,6 +90,12 @@ const MSISDN_STAGE_TYPE = "m.login.msisdn";
  * @param {string?} opts.emailSid If returning from having completed m.login.email.identity
  *     auth, the sid for the email verification session.
  *
+ * @param {function?} opts.requestEmailToken A function that takes the email address (string),
+ *     clientSecret (string), attempt number (int) and sessionId (string) and calls the
+ *     relevant requestToken function and returns the promise returned by that function.
+ *     If the resulting promise rejects, the rejection will propagate through to the
+ *     attemptAuth promise.
+ *
  */
 function InteractiveAuth(opts) {
     this._matrixClient = opts.matrixClient;
@@ -95,14 +103,17 @@ function InteractiveAuth(opts) {
     this._requestCallback = opts.doRequest;
     // startAuthStage included for backwards compat
     this._stateUpdatedCallback = opts.stateUpdated || opts.startAuthStage;
-    this._completionDeferred = null;
+    this._resolveFunc = null;
+    this._rejectFunc = null;
     this._inputs = opts.inputs || {};
+    this._requestEmailTokenCallback = opts.requestEmailToken;
 
     if (opts.sessionId) this._data.session = opts.sessionId;
     this._clientSecret = opts.clientSecret || this._matrixClient.generateClientSecret();
     this._emailSid = opts.emailSid;
     if (this._emailSid === undefined) this._emailSid = null;
 
+    this._chosenFlow = null;
     this._currentStage = null;
 }
 
@@ -115,11 +126,12 @@ InteractiveAuth.prototype = {
      *     no suitable authentication flow can be found
      */
     attemptAuth: function() {
-        this._completionDeferred = Promise.defer();
+        // This promise will be quite long-lived and will resolve when the
+        // request is authenticated and completes successfully.
+        return new Promise((resolve, reject) => {
+            this._resolveFunc = resolve;
+            this._rejectFunc = reject;
 
-        // wrap in a promise so that if _startNextAuthStage
-        // throws, it rejects the promise in a consistent way
-        return Promise.resolve().then(() => {
             // if we have no flows, try a request (we'll have
             // just a session ID in _data if resuming)
             if (!this._data.flows) {
@@ -127,7 +139,6 @@ InteractiveAuth.prototype = {
             } else {
                 this._startNextAuthStage();
             }
-            return this._completionDeferred.promise;
         });
     },
 
@@ -194,6 +205,10 @@ InteractiveAuth.prototype = {
         return params[loginType];
     },
 
+    getChosenFlow() {
+        return this._chosenFlow;
+    },
+
     /**
      * submit a new auth dict and fire off the request. This will either
      * make attemptAuth resolve/reject, or cause the startAuthStage callback
@@ -207,7 +222,7 @@ InteractiveAuth.prototype = {
      *    for requests that just poll to see if auth has been completed elsewhere.
      */
     submitAuthDict: function(authData, background) {
-        if (!this._completionDeferred) {
+        if (!this._resolveFunc) {
             throw new Error("submitAuthDict() called before attemptAuth()");
         }
 
@@ -253,30 +268,27 @@ InteractiveAuth.prototype = {
      *    This can be set to true for requests that just poll to see if auth has
      *    been completed elsewhere.
      */
-    _doRequest: function(auth, background) {
+    _doRequest: async function(auth, background) {
-        const self = this;
-
-        // hackery to make sure that synchronous exceptions end up in the catch
-        // handler (without the additional event loop entailed by q.fcall or an
-        // extra Promise.resolve().then)
-        let prom;
         try {
-            prom = this._requestCallback(auth, background);
+            const result = await this._requestCallback(auth, background);
-        } catch (e) {
+            this._resolveFunc(result);
-            prom = Promise.reject(e);
+        } catch (error) {
-        }
-
-        prom = prom.then(
-            function(result) {
-                console.log("result from request: ", result);
-                self._completionDeferred.resolve(result);
-            }, function(error) {
             // sometimes UI auth errors don't come with flows
             const errorFlows = error.data ? error.data.flows : null;
-                const haveFlows = Boolean(self._data.flows) || Boolean(errorFlows);
+            const haveFlows = Boolean(this._data.flows) || Boolean(errorFlows);
             if (error.httpStatus !== 401 || !error.data || !haveFlows) {
-                    // doesn't look like an interactive-auth failure. fail the whole lot.
+                // doesn't look like an interactive-auth failure.
-                    throw error;
+                if (!background) {
+                    this._rejectFunc(error);
+                } else {
+                    // We ignore all failures here (even non-UI auth related ones)
+                    // since we don't want to suddenly fail if the internet connection
+                    // had a blip whilst we were polling
+                    logger.log(
+                        "Background poll request failed doing UI auth: ignoring",
+                        error,
+                    );
+                }
             }
             // if the error didn't come with flows, completed flows or session ID,
             // copy over the ones we have. Synapse sometimes sends responses without
@@ -284,27 +296,46 @@ InteractiveAuth.prototype = {
             // has not yet been validated). This appears to be a Synapse bug, which
             // we workaround here.
             if (!error.data.flows && !error.data.completed && !error.data.session) {
-                    error.data.flows = self._data.flows;
+                error.data.flows = this._data.flows;
-                    error.data.completed = self._data.completed;
+                error.data.completed = this._data.completed;
-                    error.data.session = self._data.session;
+                error.data.session = this._data.session;
             }
-                self._data = error.data;
+            this._data = error.data;
-                self._startNextAuthStage();
+            this._startNextAuthStage();
-            },
+
+            if (
+                !this._emailSid &&
+                this._chosenFlow.stages.includes('m.login.email.identity')
+            ) {
+                // If we've picked a flow with email auth, we send the email
+                // now because we want the request to fail as soon as possible
+                // if the email address is not valid (ie. already taken or not
+                // registered, depending on what the operation is).
+                try {
+                    const requestTokenResult = await this._requestEmailTokenCallback(
+                        this._inputs.emailAddress,
+                        this._clientSecret,
+                        1, // TODO: Multiple send attempts?
+                        this._data.session,
                     );
-        if (!background) {
+                    this._emailSid = requestTokenResult.sid;
-            prom = prom.catch((e) => {
+                    // NB. promise is not resolved here - at some point, doRequest
-                this._completionDeferred.reject(e);
+                    // will be called again and if the user has jumped through all
-            });
+                    // the hoops correctly, auth will be complete and the request
-        } else {
+                    // will succeed.
-            // We ignore all failures here (even non-UI auth related ones)
+                    // Also, we should expose the fact that this request has compledted
-            // since we don't want to suddenly fail if the internet connection
+                    // so clients can know that the email has actually been sent.
-            // had a blip whilst we were polling
+                } catch (e) {
-            prom = prom.catch((error) => {
+                    // we failed to request an email token, so fail the request.
-                console.log("Ignoring error from UI auth: " + error);
+                    // This could be due to the email already beeing registered
-            });
+                    // (or not being registered, depending on what we're trying
+                    // to do) or it could be a network failure. Either way, pass
+                    // the failure up as the user can't complete auth if we can't
+                    // send the email, foe whatever reason.
+                    this._rejectFunc(e);
+                }
+            }
         }
-        prom.done();
     },
 
     /**
@@ -320,7 +351,7 @@ InteractiveAuth.prototype = {
         }
         this._currentStage = nextStage;
 
-        if (nextStage == 'm.login.dummy') {
+        if (nextStage === 'm.login.dummy') {
             this.submitAuthDict({
                 type: 'm.login.dummy',
             });
@@ -350,10 +381,12 @@ InteractiveAuth.prototype = {
      * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _chooseStage: function() {
-        const flow = this._chooseFlow();
+        if (this._chosenFlow === null) {
-        console.log("Active flow => %s", JSON.stringify(flow));
+            this._chosenFlow = this._chooseFlow();
-        const nextStage = this._firstUncompletedStage(flow);
+        }
-        console.log("Next stage: %s", nextStage);
+        logger.log("Active flow => %s", JSON.stringify(this._chosenFlow));
+        const nextStage = this._firstUncompletedStage(this._chosenFlow);
+        logger.log("Next stage: %s", nextStage);
         return nextStage;
     },
 

src/models/event-timeline-set.js

@@ -20,6 +20,8 @@ limitations under the License.
 const EventEmitter = require("events").EventEmitter;
 const utils = require("../utils");
 const EventTimeline = require("./event-timeline");
+import logger from '../../src/logger';
+import Relations from './relations';
 
 // var DEBUG = false;
 const DEBUG = true;
@@ -27,7 +29,7 @@ const DEBUG = true;
 let debuglog;
 if (DEBUG) {
     // using bind means that we get to keep useful line numbers in the console
-    debuglog = console.log.bind(console);
+    debuglog = logger.log.bind(logger);
 } else {
     debuglog = function() {};
 }
@@ -54,22 +56,38 @@ if (DEBUG) {
  * map from event_id to timeline and index.
  *
  * @constructor
- * @param {?Room} room      the optional room for this timelineSet
+ * @param {?Room} room
- * @param {Object} opts     hash of options inherited from Room.
+ * Room for this timelineSet. May be null for non-room cases, such as the
- *      opts.timelineSupport gives whether timeline support is enabled
+ * notification timeline.
- *      opts.filter is the filter object, if any, for this timelineSet.
+ * @param {Object} opts Options inherited from Room.
+ *
+ * @param {boolean} [opts.timelineSupport = false]
+ * Set to true to enable improved timeline support.
+ * @param {Object} [opts.filter = null]
+ * The filter object, if any, for this timelineSet.
+ * @param {boolean} [opts.unstableClientRelationAggregation = false]
+ * Optional. Set to true to enable client-side aggregation of event relations
+ * via `getRelationsForEvent`.
+ * This feature is currently unstable and the API may change without notice.
  */
 function EventTimelineSet(room, opts) {
     this.room = room;
 
     this._timelineSupport = Boolean(opts.timelineSupport);
     this._liveTimeline = new EventTimeline(this);
+    this._unstableClientRelationAggregation = !!opts.unstableClientRelationAggregation;
 
     // just a list - *not* ordered.
     this._timelines = [this._liveTimeline];
     this._eventIdToTimeline = {};
 
     this._filter = opts.filter || null;
+
+    if (this._unstableClientRelationAggregation) {
+        // A tree of objects to access a set of relations for an event, as in:
+        // this._relations[relatesToEventId][relationType][relationEventType]
+        this._relations = {};
+    }
 }
 utils.inherits(EventTimelineSet, EventEmitter);
 
@@ -405,7 +423,7 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
         }
 
         // time to join the timelines.
-        console.info("Already have timeline for " + eventId +
+        logger.info("Already have timeline for " + eventId +
                      " - joining timeline " + timeline + " to " +
                      existingTimeline);
 
@@ -413,25 +431,30 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
         const existingIsLive = existingTimeline === this._liveTimeline;
         const timelineIsLive = timeline === this._liveTimeline;
 
-        if (direction === EventTimeline.BACKWARDS && existingIsLive) {
+        const backwardsIsLive = direction === EventTimeline.BACKWARDS && existingIsLive;
+        const forwardsIsLive = direction === EventTimeline.FORWARDS && timelineIsLive;
+
+        if (backwardsIsLive || forwardsIsLive) {
             // The live timeline should never be spliced into a non-live position.
-            console.warn(
+            // We use independent logging to better discover the problem at a glance.
+            logger.warn({backwardsIsLive, forwardsIsLive}); // debugging
+            if (backwardsIsLive) {
+                logger.warn(
                     "Refusing to set a preceding existingTimeLine on our " +
                     "timeline as the existingTimeLine is live (" + existingTimeline + ")",
                 );
-        } else {
-            timeline.setNeighbouringTimeline(existingTimeline, direction);
             }
-
+            if (forwardsIsLive) {
-        if (inverseDirection === EventTimeline.BACKWARDS && timelineIsLive) {
+                logger.warn(
-            // The live timeline should never be spliced into a non-live position.
-            console.warn(
                     "Refusing to set our preceding timeline on a existingTimeLine " +
                     "as our timeline is live (" + timeline + ")",
                 );
-        } else {
-            existingTimeline.setNeighbouringTimeline(timeline, inverseDirection);
             }
+            continue; // abort splicing - try next event
+        }
+
+        timeline.setNeighbouringTimeline(existingTimeline, direction);
+        existingTimeline.setNeighbouringTimeline(timeline, inverseDirection);
 
         timeline = existingTimeline;
         didUpdate = true;
@@ -441,6 +464,14 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
     // new information, we update the pagination token for whatever
     // timeline we ended up on.
     if (lastEventWasNew || !didUpdate) {
+        if (direction === EventTimeline.FORWARDS && timeline === this._liveTimeline) {
+            logger.warn({lastEventWasNew, didUpdate}); // for debugging
+            logger.warn(
+                `Refusing to set forwards pagination token of live timeline ` +
+                `${timeline} to ${paginationToken}`,
+            );
+            return;
+        }
         timeline.setPaginationToken(paginationToken, direction);
     }
 };
@@ -510,6 +541,9 @@ EventTimelineSet.prototype.addEventToTimeline = function(event, timeline,
     timeline.addEvent(event, toStartOfTimeline);
     this._eventIdToTimeline[eventId] = timeline;
 
+    this.setRelationsTarget(event);
+    this.aggregateRelations(event);
+
     const data = {
         timeline: timeline,
         liveEvent: !toStartOfTimeline && timeline == this._liveTimeline,
@@ -644,6 +678,122 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
     return null;
 };
 
+/**
+ * Get a collection of relations to a given event in this timeline set.
+ *
+ * @param {String} eventId
+ * The ID of the event that you'd like to access relation events for.
+ * For example, with annotations, this would be the ID of the event being annotated.
+ * @param {String} relationType
+ * The type of relation involved, such as "m.annotation", "m.reference", "m.replace", etc.
+ * @param {String} eventType
+ * The relation event's type, such as "m.reaction", etc.
+ *
+ * @returns {Relations}
+ * A container for relation events.
+ */
+EventTimelineSet.prototype.getRelationsForEvent = function(
+    eventId, relationType, eventType,
+) {
+    if (!this._unstableClientRelationAggregation) {
+        throw new Error("Client-side relation aggregation is disabled");
+    }
+
+    if (!eventId || !relationType || !eventType) {
+        throw new Error("Invalid arguments for `getRelationsForEvent`");
+    }
+
+    // debuglog("Getting relations for: ", eventId, relationType, eventType);
+
+    const relationsForEvent = this._relations[eventId] || {};
+    const relationsWithRelType = relationsForEvent[relationType] || {};
+    return relationsWithRelType[eventType];
+};
+
+/**
+ * Set an event as the target event if any Relations exist for it already
+ *
+ * @param {MatrixEvent} event
+ * The event to check as relation target.
+ */
+EventTimelineSet.prototype.setRelationsTarget = function(event) {
+    if (!this._unstableClientRelationAggregation) {
+        return;
+    }
+
+    const relationsForEvent = this._relations[event.getId()];
+    if (!relationsForEvent) {
+        return;
+    }
+    // don't need it for non m.replace relations for now
+    const relationsWithRelType = relationsForEvent["m.replace"];
+    if (!relationsWithRelType) {
+        return;
+    }
+    // only doing replacements for messages for now (e.g. edits)
+    const relationsWithEventType = relationsWithRelType["m.room.message"];
+
+    if (relationsWithEventType) {
+        relationsWithEventType.setTargetEvent(event);
+    }
+};
+
+/**
+ * Add relation events to the relevant relation collection.
+ *
+ * @param {MatrixEvent} event
+ * The new relation event to be aggregated.
+ */
+EventTimelineSet.prototype.aggregateRelations = function(event) {
+    if (!this._unstableClientRelationAggregation) {
+        return;
+    }
+
+    // If the event is currently encrypted, wait until it has been decrypted.
+    if (event.isBeingDecrypted()) {
+        event.once("Event.decrypted", () => {
+            this.aggregateRelations(event);
+        });
+        return;
+    }
+
+    const relation = event.getRelation();
+    if (!relation) {
+        return;
+    }
+
+    const relatesToEventId = relation.event_id;
+    const relationType = relation.rel_type;
+    const eventType = event.getType();
+
+    // debuglog("Aggregating relation: ", event.getId(), eventType, relation);
+
+    let relationsForEvent = this._relations[relatesToEventId];
+    if (!relationsForEvent) {
+        relationsForEvent = this._relations[relatesToEventId] = {};
+    }
+    let relationsWithRelType = relationsForEvent[relationType];
+    if (!relationsWithRelType) {
+        relationsWithRelType = relationsForEvent[relationType] = {};
+    }
+    let relationsWithEventType = relationsWithRelType[eventType];
+
+    if (!relationsWithEventType) {
+        relationsWithEventType = relationsWithRelType[eventType] = new Relations(
+            relationType,
+            eventType,
+            this.room,
+        );
+        const relatesToEvent = this.findEventById(relatesToEventId);
+        if (relatesToEvent) {
+            relationsWithEventType.setTargetEvent(relatesToEvent);
+            relatesToEvent.emit("Event.relationsCreated", relationType, eventType);
+        }
+    }
+
+    relationsWithEventType.addEvent(event);
+};
+
 /**
  * The EventTimelineSet class.
  */

src/models/event.js

@@ -24,6 +24,7 @@ limitations under the License.
 import Promise from 'bluebird';
 import {EventEmitter} from 'events';
 import utils from '../utils.js';
+import logger from '../../src/logger';
 
 /**
  * Enum for event statuses.
@@ -50,6 +51,12 @@ module.exports.EventStatus = {
 };
 
 const interns = {};
+function intern(str) {
+    if (!interns[str]) {
+        interns[str] = str;
+    }
+    return interns[str];
+}
 
 /**
  * Construct a Matrix Event object
@@ -87,20 +94,25 @@ module.exports.MatrixEvent = function MatrixEvent(
         if (!event[prop]) {
             return;
         }
-        if (!interns[event[prop]]) {
+        event[prop] = intern(event[prop]);
-            interns[event[prop]] = event[prop];
-        }
-        event[prop] = interns[event[prop]];
     });
 
     ["membership", "avatar_url", "displayname"].forEach((prop) => {
         if (!event.content || !event.content[prop]) {
             return;
         }
-        if (!interns[event.content[prop]]) {
+        event.content[prop] = intern(event.content[prop]);
-            interns[event.content[prop]] = event.content[prop];
+    });
+
+    ["rel_type"].forEach((prop) => {
+        if (
+            !event.content ||
+            !event.content["m.relates_to"] ||
+            !event.content["m.relates_to"][prop]
+        ) {
+            return;
         }
-        event.content[prop] = interns[event.content[prop]];
+        event.content["m.relates_to"][prop] = intern(event.content["m.relates_to"][prop]);
     });
 
     this.event = event || {};
@@ -111,6 +123,8 @@ module.exports.MatrixEvent = function MatrixEvent(
     this.error = null;
     this.forwardLooking = true;
     this._pushActions = null;
+    this._replacingEvent = null;
+    this._locallyRedacted = false;
 
     this._clearEvent = {};
 
@@ -209,12 +223,33 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     /**
-     * Get the (decrypted, if necessary) event content JSON.
+     * Get the (decrypted, if necessary) event content JSON, even if the event
+     * was replaced by another event.
+     *
+     * @return {Object} The event content JSON, or an empty object.
+     */
+    getOriginalContent: function() {
+        if (this._locallyRedacted) {
+            return {};
+        }
+        return this._clearEvent.content || this.event.content || {};
+    },
+
+    /**
+     * Get the (decrypted, if necessary) event content JSON,
+     * or the content from the replacing event, if any.
+     * See `makeReplaced`.
      *
      * @return {Object} The event content JSON, or an empty object.
      */
     getContent: function() {
-        return this._clearEvent.content || this.event.content || {};
+        if (this._locallyRedacted) {
+            return {};
+        } else if (this._replacingEvent) {
+            return this._replacingEvent.getContent()["m.new_content"] || {};
+        } else {
+            return this.getOriginalContent();
+        }
     },
 
     /**
@@ -367,7 +402,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         // new info.
         //
         if (this._decryptionPromise) {
-            console.log(
+            logger.log(
                 `Event ${this.getId()} already being decrypted; queueing a retry`,
             );
             this._retryDecryption = true;
@@ -441,7 +476,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
                 if (e.name !== "DecryptionError") {
                     // not a decryption error: log the whole exception as an error
                     // (and don't bother with a retry)
-                    console.error(
+                    logger.error(
                         `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
                     );
                     this._decryptionPromise = null;
@@ -467,7 +502,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
                 //
                 if (this._retryDecryption) {
                     // decryption error, but we have a retry queued.
-                    console.log(
+                    logger.log(
                         `Got error decrypting event (id=${this.getId()}: ` +
                         `${e}), but retrying`,
                     );
@@ -476,7 +511,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
 
                 // decryption error, no retries queued. Warn about the error and
                 // set it to m.bad.encrypted.
-                console.warn(
+                logger.warn(
                     `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
                 );
 
@@ -637,6 +672,27 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return this.event.unsigned || {};
     },
 
+    unmarkLocallyRedacted: function() {
+        const value = this._locallyRedacted;
+        this._locallyRedacted = false;
+        if (this.event.unsigned) {
+            this.event.unsigned.redacted_because = null;
+        }
+        return value;
+    },
+
+    markLocallyRedacted: function(redactionEvent) {
+        if (this._locallyRedacted) {
+            return;
+        }
+        this.emit("Event.beforeRedaction", this, redactionEvent);
+        this._locallyRedacted = true;
+        if (!this.event.unsigned) {
+            this.event.unsigned = {};
+        }
+        this.event.unsigned.redacted_because = redactionEvent.event;
+    },
+
     /**
      * Update the content of an event in the same way it would be by the server
      * if it were redacted before it was sent to us
@@ -650,6 +706,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             throw new Error("invalid redaction_event in makeRedacted");
         }
 
+        this._locallyRedacted = false;
+
+        this.emit("Event.beforeRedaction", this, redaction_event);
+
+        this._replacingEvent = null;
         // we attempt to replicate what we would see from the server if
         // the event had been redacted before we saw it.
         //
@@ -717,7 +778,131 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     handleRemoteEcho: function(event) {
         this.event = event;
         // successfully sent.
-        this.status = null;
+        this.setStatus(null);
+    },
+
+    /**
+     * Whether the event is in any phase of sending, send failure, waiting for
+     * remote echo, etc.
+     *
+     * @return {boolean}
+     */
+    isSending() {
+        return !!this.status;
+    },
+
+    /**
+     * Update the event's sending status and emit an event as well.
+     *
+     * @param {String} status The new status
+     */
+    setStatus(status) {
+        this.status = status;
+        this.emit("Event.status", this, status);
+    },
+
+    /**
+     * Get whether the event is a relation event, and of a given type if
+     * `relType` is passed in.
+     *
+     * @param {string?} relType if given, checks that the relation is of the
+     * given type
+     * @return {boolean}
+     */
+    isRelation(relType = undefined) {
+        // Relation info is lifted out of the encrypted content when sent to
+        // encrypted rooms, so we have to check `getWireContent` for this.
+        const content = this.getWireContent();
+        const relation = content && content["m.relates_to"];
+        return relation && relation.rel_type && relation.event_id &&
+            ((relType && relation.rel_type === relType) || !relType);
+    },
+
+    /**
+     * Get relation info for the event, if any.
+     *
+     * @return {Object}
+     */
+    getRelation() {
+        if (!this.isRelation()) {
+            return null;
+        }
+        return this.getWireContent()["m.relates_to"];
+    },
+
+    /**
+     * Set an event that replaces the content of this event, through an m.replace relation.
+     *
+     * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
+     */
+    makeReplaced(newEvent) {
+        if (this.isRedacted()) {
+            return;
+        }
+        if (this._replacingEvent !== newEvent) {
+            this._replacingEvent = newEvent;
+            this.emit("Event.replaced", this);
+        }
+    },
+
+    /**
+     * Returns the status of the event, or the replacing event in case `makeReplace` has been called.
+     *
+     * @return {EventStatus}
+     */
+    replacementOrOwnStatus() {
+        if (this._replacingEvent) {
+            return this._replacingEvent.status;
+        } else {
+            return this.status;
+        }
+    },
+
+    /**
+     * Returns the event ID of the event replacing the content of this event, if any.
+     *
+     * @return {string?}
+     */
+    replacingEventId() {
+        return this._replacingEvent && this._replacingEvent.getId();
+    },
+
+    /**
+     * Returns the event replacing the content of this event, if any.
+     *
+     * @return {MatrixEvent?}
+     */
+    replacingEvent() {
+        return this._replacingEvent;
+    },
+
+    /**
+     * Summarise the event as JSON for debugging. If encrypted, include both the
+     * decrypted and encrypted view of the event. This is named `toJSON` for use
+     * with `JSON.stringify` which checks objects for functions named `toJSON`
+     * and will call them to customise the output if they are defined.
+     *
+     * @return {Object}
+     */
+    toJSON() {
+        const event = {
+            type: this.getType(),
+            sender: this.getSender(),
+            content: this.getContent(),
+            event_id: this.getId(),
+            origin_server_ts: this.getTs(),
+            unsigned: this.getUnsigned(),
+            room_id: this.getRoomId(),
+        };
+
+        if (!this.isEncrypted()) {
+            return event;
+        }
+
+        return {
+            decrypted: event,
+            encrypted: this.event,
+        };
     },
 });
 

src/models/relations.js

@@ -0,0 +1,337 @@
+/*
+Copyright 2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import EventEmitter from 'events';
+import { EventStatus } from '../../lib/models/event';
+
+/**
+ * A container for relation events that supports easy access to common ways of
+ * aggregating such events. Each instance holds events that of a single relation
+ * type and event type. All of the events also relate to the same original event.
+ *
+ * The typical way to get one of these containers is via
+ * EventTimelineSet#getRelationsForEvent.
+ */
+export default class Relations extends EventEmitter {
+    /**
+     * @param {String} relationType
+     * The type of relation involved, such as "m.annotation", "m.reference",
+     * "m.replace", etc.
+     * @param {String} eventType
+     * The relation event's type, such as "m.reaction", etc.
+     * @param {?Room} room
+     * Room for this container. May be null for non-room cases, such as the
+     * notification timeline.
+     */
+    constructor(relationType, eventType, room) {
+        super();
+        this.relationType = relationType;
+        this.eventType = eventType;
+        this._relations = new Set();
+        this._annotationsByKey = {};
+        this._annotationsBySender = {};
+        this._sortedAnnotationsByKey = [];
+        this._targetEvent = null;
+    }
+
+    /**
+     * Add relation events to this collection.
+     *
+     * @param {MatrixEvent} event
+     * The new relation event to be added.
+     */
+    addEvent(event) {
+        if (this._relations.has(event)) {
+            return;
+        }
+
+        const relation = event.getRelation();
+        if (!relation) {
+            console.error("Event must have relation info");
+            return;
+        }
+
+        const relationType = relation.rel_type;
+        const eventType = event.getType();
+
+        if (this.relationType !== relationType || this.eventType !== eventType) {
+            console.error("Event relation info doesn't match this container");
+            return;
+        }
+
+        // If the event is in the process of being sent, listen for cancellation
+        // so we can remove the event from the collection.
+        if (event.isSending()) {
+            event.on("Event.status", this._onEventStatus);
+        }
+
+        this._relations.add(event);
+
+        if (this.relationType === "m.annotation") {
+            this._addAnnotationToAggregation(event);
+        } else if (this.relationType === "m.replace" && this._targetEvent) {
+            this._targetEvent.makeReplaced(this.getLastReplacement());
+        }
+
+        event.on("Event.beforeRedaction", this._onBeforeRedaction);
+
+        this.emit("Relations.add", event);
+    }
+
+    /**
+     * Remove relation event from this collection.
+     *
+     * @param {MatrixEvent} event
+     * The relation event to remove.
+     */
+    _removeEvent(event) {
+        if (!this._relations.has(event)) {
+            return;
+        }
+
+        const relation = event.getRelation();
+        if (!relation) {
+            console.error("Event must have relation info");
+            return;
+        }
+
+        const relationType = relation.rel_type;
+        const eventType = event.getType();
+
+        if (this.relationType !== relationType || this.eventType !== eventType) {
+            console.error("Event relation info doesn't match this container");
+            return;
+        }
+
+        this._relations.delete(event);
+
+        if (this.relationType === "m.annotation") {
+            this._removeAnnotationFromAggregation(event);
+        } else if (this.relationType === "m.replace" && this._targetEvent) {
+            this._targetEvent.makeReplaced(this.getLastReplacement());
+        }
+
+        this.emit("Relations.remove", event);
+    }
+
+    /**
+     * Listens for event status changes to remove cancelled events.
+     *
+     * @param {MatrixEvent} event The event whose status has changed
+     * @param {EventStatus} status The new status
+     */
+    _onEventStatus = (event, status) => {
+        if (!event.isSending()) {
+            // Sending is done, so we don't need to listen anymore
+            event.removeListener("Event.status", this._onEventStatus);
+            return;
+        }
+        if (status !== EventStatus.CANCELLED) {
+            return;
+        }
+        // Event was cancelled, remove from the collection
+        event.removeListener("Event.status", this._onEventStatus);
+        this._removeEvent(event);
+    }
+
+    /**
+     * Get all relation events in this collection.
+     *
+     * These are currently in the order of insertion to this collection, which
+     * won't match timeline order in the case of scrollback.
+     * TODO: Tweak `addEvent` to insert correctly for scrollback.
+     *
+     * @return {Array}
+     * Relation events in insertion order.
+     */
+    getRelations() {
+        return [...this._relations];
+    }
+
+    _addAnnotationToAggregation(event) {
+        const { key } = event.getRelation();
+        if (!key) {
+            return;
+        }
+
+        let eventsForKey = this._annotationsByKey[key];
+        if (!eventsForKey) {
+            eventsForKey = this._annotationsByKey[key] = new Set();
+            this._sortedAnnotationsByKey.push([key, eventsForKey]);
+        }
+        // Add the new event to the set for this key
+        eventsForKey.add(event);
+        // Re-sort the [key, events] pairs in descending order of event count
+        this._sortedAnnotationsByKey.sort((a, b) => {
+            const aEvents = a[1];
+            const bEvents = b[1];
+            return bEvents.size - aEvents.size;
+        });
+
+        const sender = event.getSender();
+        let eventsFromSender = this._annotationsBySender[sender];
+        if (!eventsFromSender) {
+            eventsFromSender = this._annotationsBySender[sender] = new Set();
+        }
+        // Add the new event to the set for this sender
+        eventsFromSender.add(event);
+    }
+
+    _removeAnnotationFromAggregation(event) {
+        const { key } = event.getRelation();
+        if (!key) {
+            return;
+        }
+
+        const eventsForKey = this._annotationsByKey[key];
+        if (eventsForKey) {
+            eventsForKey.delete(event);
+
+            // Re-sort the [key, events] pairs in descending order of event count
+            this._sortedAnnotationsByKey.sort((a, b) => {
+                const aEvents = a[1];
+                const bEvents = b[1];
+                return bEvents.size - aEvents.size;
+            });
+        }
+
+        const sender = event.getSender();
+        const eventsFromSender = this._annotationsBySender[sender];
+        if (eventsFromSender) {
+            eventsFromSender.delete(event);
+        }
+    }
+
+    /**
+     * For relations that have been redacted, we want to remove them from
+     * aggregation data sets and emit an update event.
+     *
+     * To do so, we listen for `Event.beforeRedaction`, which happens:
+     *   - after the server accepted the redaction and remote echoed back to us
+     *   - before the original event has been marked redacted in the client
+     *
+     * @param {MatrixEvent} redactedEvent
+     * The original relation event that is about to be redacted.
+     */
+    _onBeforeRedaction = (redactedEvent) => {
+        if (!this._relations.has(redactedEvent)) {
+            return;
+        }
+
+        this._relations.delete(redactedEvent);
+
+        if (this.relationType === "m.annotation") {
+            // Remove the redacted annotation from aggregation by key
+            this._removeAnnotationFromAggregation(redactedEvent);
+        } else if (this.relationType === "m.replace" && this._targetEvent) {
+            this._targetEvent.makeReplaced(this.getLastReplacement());
+        }
+
+        redactedEvent.removeListener("Event.beforeRedaction", this._onBeforeRedaction);
+
+        // Dispatch a redaction event on this collection. `setTimeout` is used
+        // to wait until the next event loop iteration by which time the event
+        // has actually been marked as redacted.
+        setTimeout(() => {
+            this.emit("Relations.redaction");
+        }, 0);
+    }
+
+    /**
+     * Get all events in this collection grouped by key and sorted by descending
+     * event count in each group.
+     *
+     * This is currently only supported for the annotation relation type.
+     *
+     * @return {Array}
+     * An array of [key, events] pairs sorted by descending event count.
+     * The events are stored in a Set (which preserves insertion order).
+     */
+    getSortedAnnotationsByKey() {
+        if (this.relationType !== "m.annotation") {
+            // Other relation types are not grouped currently.
+            return null;
+        }
+
+        return this._sortedAnnotationsByKey;
+    }
+
+    /**
+     * Get all events in this collection grouped by sender.
+     *
+     * This is currently only supported for the annotation relation type.
+     *
+     * @return {Object}
+     * An object with each relation sender as a key and the matching Set of
+     * events for that sender as a value.
+     */
+    getAnnotationsBySender() {
+        if (this.relationType !== "m.annotation") {
+            // Other relation types are not grouped currently.
+            return null;
+        }
+
+        return this._annotationsBySender;
+    }
+
+    /**
+     * Returns the most recent (and allowed) m.replace relation, if any.
+     *
+     * This is currently only supported for the m.replace relation type,
+     * once the target event is known, see `addEvent`.
+     *
+     * @return {MatrixEvent?}
+     */
+    getLastReplacement() {
+        if (this.relationType !== "m.replace") {
+            // Aggregating on last only makes sense for this relation type
+            return null;
+        }
+        if (!this._targetEvent) {
+            // Don't know which replacements to accept yet.
+            // This method shouldn't be called before the original
+            // event is known anyway.
+            return null;
+        }
+        return this.getRelations().reduce((last, event) => {
+            if (event.getSender() !== this._targetEvent.getSender()) {
+                return last;
+            }
+            if (last && last.getTs() > event.getTs()) {
+                return last;
+            }
+            return event;
+        }, null);
+    }
+
+    /*
+     * @param {MatrixEvent} targetEvent the event the relations are related to.
+     */
+    setTargetEvent(event) {
+        if (this._targetEvent) {
+            return;
+        }
+        this._targetEvent = event;
+        if (this.relationType === "m.replace") {
+            const replacement = this.getLastReplacement();
+            // this is the initial update, so only call it if we already have something
+            // to not emit Event.replaced needlessly
+            if (replacement) {
+                this._targetEvent.makeReplaced(replacement);
+            }
+        }
+    }
+}

src/models/room-state.js

@@ -21,6 +21,7 @@ const EventEmitter = require("events").EventEmitter;
 
 const utils = require("../utils");
 const RoomMember = require("./room-member");
+import logger from '../../src/logger';
 
 // possible statuses for out-of-band member loading
 const OOB_STATUS_NOTSTARTED = 1;
@@ -447,7 +448,7 @@ RoomState.prototype.clearOutOfBandMembers = function() {
             delete this.members[userId];
         }
     });
-    console.log(`LL: RoomState removed ${count} members...`);
+    logger.log(`LL: RoomState removed ${count} members...`);
     this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
 };
 
@@ -456,11 +457,11 @@ RoomState.prototype.clearOutOfBandMembers = function() {
  * @param {MatrixEvent[]} stateEvents array of membership state events
  */
 RoomState.prototype.setOutOfBandMembers = function(stateEvents) {
-    console.log(`LL: RoomState about to set ${stateEvents.length} OOB members ...`);
+    logger.log(`LL: RoomState about to set ${stateEvents.length} OOB members ...`);
     if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
         return;
     }
-    console.log(`LL: RoomState put in OOB_STATUS_FINISHED state ...`);
+    logger.log(`LL: RoomState put in OOB_STATUS_FINISHED state ...`);
     this._oobMemberFlags.status = OOB_STATUS_FINISHED;
     stateEvents.forEach((e) => this._setOutOfBandMember(e));
 };

src/models/room.js

@@ -1,6 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2018 New Vector Ltd
+Copyright 2018, 2019 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -29,6 +29,7 @@ const ContentRepo = require("../content-repo");
 const EventTimeline = require("./event-timeline");
 const EventTimelineSet = require("./event-timeline-set");
 
+import logger from '../../src/logger';
 import ReEmitter from '../ReEmitter';
 
 // These constants are used as sane defaults when the homeserver doesn't support
@@ -38,7 +39,7 @@ import ReEmitter from '../ReEmitter';
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
 const KNOWN_SAFE_ROOM_VERSION = '1';
-const SAFE_ROOM_VERSIONS = ['1', '2'];
+const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4'];
 
 function synthesizeReceipt(userId, event, receiptType) {
     // console.log("synthesizing receipt for "+event.getId());
@@ -92,9 +93,12 @@ function synthesizeReceipt(userId, event, receiptType) {
  * "<b>detached</b>", pending messages will appear in a separate list,
  * accessbile via {@link module:models/room#getPendingEvents}. Default:
  * "chronological".
- *
  * @param {boolean} [opts.timelineSupport = false] Set to true to enable improved
  * timeline support.
+ * @param {boolean} [opts.unstableClientRelationAggregation = false]
+ * Optional. Set to true to enable client-side aggregation of event relations
+ * via `EventTimelineSet#getRelationsForEvent`.
+ * This feature is currently unstable and the API may change without notice.
  *
  * @prop {string} roomId The ID of this room.
  * @prop {string} name The human-readable display name for this room.
@@ -206,7 +210,7 @@ utils.inherits(Room, EventEmitter);
 Room.prototype.getVersion = function() {
     const createEvent = this.currentState.getStateEvents("m.room.create", "");
     if (!createEvent) {
-        console.warn("Room " + this.room_id + " does not have an m.room.create event");
+        logger.warn("Room " + this.room_id + " does not have an m.room.create event");
         return '1';
     }
     const ver = createEvent.getContent()['room_version'];
@@ -259,9 +263,36 @@ Room.prototype.getRecommendedVersion = async function() {
         }
     }
 
+    let result = this._checkVersionAgainstCapability(versionCap);
+    if (result.urgent && result.needsUpgrade) {
+        // Something doesn't feel right: we shouldn't need to update
+        // because the version we're on should be in the protocol's
+        // namespace. This usually means that the server was updated
+        // before the client was, making us think the newest possible
+        // room version is not stable. As a solution, we'll refresh
+        // the capability we're using to determine this.
+        logger.warn(
+            "Refreshing room version capability because the server looks " +
+            "to be supporting a newer room version we don't know about.",
+        );
+
+        const caps = await this._client.getCapabilities(true);
+        versionCap = caps["m.room_versions"];
+        if (!versionCap) {
+            logger.warn("No room version capability - assuming upgrade required.");
+            return result;
+        } else {
+            result = this._checkVersionAgainstCapability(versionCap);
+        }
+    }
+
+    return result;
+};
+
+Room.prototype._checkVersionAgainstCapability = function(versionCap) {
     const currentVersion = this.getVersion();
-    console.log(`[${this.roomId}] Current version: ${currentVersion}`);
+    logger.log(`[${this.roomId}] Current version: ${currentVersion}`);
-    console.log(`[${this.roomId}] Version capability: `, versionCap);
+    logger.log(`[${this.roomId}] Version capability: `, versionCap);
 
     const result = {
         version: currentVersion,
@@ -270,7 +301,7 @@ Room.prototype.getRecommendedVersion = async function() {
     };
 
     // If the room is on the default version then nothing needs to change
-    if (currentVersion === versionCap.default) return Promise.resolve(result);
+    if (currentVersion === versionCap.default) return result;
 
     const stableVersions = Object.keys(versionCap.available)
         .filter((v) => versionCap.available[v] === 'stable');
@@ -283,16 +314,16 @@ Room.prototype.getRecommendedVersion = async function() {
         result.needsUpgrade = true;
         result.urgent = !!this.getVersion().match(/^[0-9]+[0-9.]*$/g);
         if (result.urgent) {
-            console.warn(`URGENT upgrade required on ${this.roomId}`);
+            logger.warn(`URGENT upgrade required on ${this.roomId}`);
         } else {
-            console.warn(`Non-urgent upgrade required on ${this.roomId}`);
+            logger.warn(`Non-urgent upgrade required on ${this.roomId}`);
         }
-        return Promise.resolve(result);
+        return result;
     }
 
     // The room is on a stable, but non-default, version by this point.
     // No upgrade needed.
-    return Promise.resolve(result);
+    return result;
 };
 
 /**
@@ -468,7 +499,7 @@ Room.prototype._loadMembers = async function() {
     if (rawMembersEvents === null) {
         fromServer = true;
         rawMembersEvents = await this._loadMembersFromServer();
-        console.log(`LL: got ${rawMembersEvents.length} ` +
+        logger.log(`LL: got ${rawMembersEvents.length} ` +
             `members from server for room ${this.roomId}`);
     }
     const memberEvents = rawMembersEvents.map(this._client.getEventMapper());
@@ -496,7 +527,7 @@ Room.prototype.loadMembersIfNeeded = function() {
     const inMemoryUpdate = this._loadMembers().then((result) => {
         this.currentState.setOutOfBandMembers(result.memberEvents);
         // now the members are loaded, start to track the e2e devices if needed
-        if (this._client.isRoomEncrypted(this.roomId)) {
+        if (this._client.isCryptoEnabled() && this._client.isRoomEncrypted(this.roomId)) {
             this._client._crypto.trackRoomDevices(this.roomId);
         }
         return result.fromServer;
@@ -512,21 +543,21 @@ Room.prototype.loadMembersIfNeeded = function() {
             const oobMembers = this.currentState.getMembers()
                 .filter((m) => m.isOutOfBand())
                 .map((m) => m.events.member.event);
-            console.log(`LL: telling store to write ${oobMembers.length}`
+            logger.log(`LL: telling store to write ${oobMembers.length}`
                 + ` members for room ${this.roomId}`);
             const store = this._client.store;
             return store.setOutOfBandMembers(this.roomId, oobMembers)
                 // swallow any IDB error as we don't want to fail
                 // because of this
                 .catch((err) => {
-                    console.log("LL: storing OOB room members failed, oh well",
+                    logger.log("LL: storing OOB room members failed, oh well",
                         err);
                 });
         }
     }).catch((err) => {
         // as this is not awaited anywhere,
         // at least show the error in the console
-        console.error(err);
+        logger.error(err);
     });
 
     this._membersPromise = inMemoryUpdate;
@@ -552,9 +583,9 @@ Room.prototype.clearLoadedMembersIfNeeded = async function() {
  */
 Room.prototype._cleanupAfterLeaving = function() {
     this.clearLoadedMembersIfNeeded().catch((err) => {
-        console.error(`error after clearing loaded members from ` +
+        logger.error(`error after clearing loaded members from ` +
             `room ${this.roomId} after leaving`);
-        console.dir(err);
+        logger.log(err);
     });
 };
 
@@ -598,6 +629,11 @@ Room.prototype._fixUpLegacyTimelineFields = function() {
 
 /**
  * Returns whether there are any devices in the room that are unverified
+ *
+ * Note: Callers should first check if crypto is enabled on this device. If it is
+ * disabled, then we aren't tracking room devices at all, so we can't answer this, and an
+ * error will be thrown.
+ *
  * @return {bool} the result
  */
 Room.prototype.hasUnverifiedDevices = async function() {
@@ -996,7 +1032,6 @@ Room.prototype.removeFilteredTimelineSet = function(filter) {
  * @private
  */
 Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
-    let i;
     if (event.getType() === "m.room.redaction") {
         const redactId = event.event.redacts;
 
@@ -1030,7 +1065,7 @@ Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
     }
 
     // add to our timeline sets
-    for (i = 0; i < this._timelineSets.length; i++) {
+    for (let i = 0; i < this._timelineSets.length; i++) {
         this._timelineSets[i].addLiveEvent(event, duplicateStrategy);
     }
 
@@ -1094,10 +1129,26 @@ Room.prototype.addPendingEvent = function(event, txnId) {
 
     if (this._opts.pendingEventOrdering == "detached") {
         if (this._pendingEventList.some((e) => e.status === EventStatus.NOT_SENT)) {
-            console.warn("Setting event as NOT_SENT due to messages in the same state");
+            logger.warn("Setting event as NOT_SENT due to messages in the same state");
-            event.status = EventStatus.NOT_SENT;
+            event.setStatus(EventStatus.NOT_SENT);
         }
         this._pendingEventList.push(event);
+
+        if (event.isRelation()) {
+            // For pending events, add them to the relations collection immediately.
+            // (The alternate case below already covers this as part of adding to
+            // the timeline set.)
+            this._aggregateNonLiveRelation(event);
+        }
+
+        if (event.getType() === "m.room.redaction") {
+            const redactId = event.event.redacts;
+            const redactedEvent = this.getUnfilteredTimelineSet().findEventById(redactId);
+            if (redactedEvent) {
+                redactedEvent.markLocallyRedacted(event);
+                this.emit("Room.redaction", event, this);
+            }
+        }
     } else {
         for (let i = 0; i < this._timelineSets.length; i++) {
             const timelineSet = this._timelineSets[i];
@@ -1115,6 +1166,30 @@ Room.prototype.addPendingEvent = function(event, txnId) {
 
     this.emit("Room.localEchoUpdated", event, this, null, null);
 };
+/**
+ * Used to aggregate the local echo for a relation, and also
+ * for re-applying a relation after it's redaction has been cancelled,
+ * as the local echo for the redaction of the relation would have
+ * un-aggregated the relation. Note that this is different from regular messages,
+ * which are just kept detached for their local echo.
+ *
+ * Also note that live events are aggregated in the live EventTimelineSet.
+ * @param {module:models/event.MatrixEvent} event the relation event that needs to be aggregated.
+ */
+Room.prototype._aggregateNonLiveRelation = function(event) {
+    // TODO: We should consider whether this means it would be a better
+    // design to lift the relations handling up to the room instead.
+    for (let i = 0; i < this._timelineSets.length; i++) {
+        const timelineSet = this._timelineSets[i];
+        if (timelineSet.getFilter()) {
+            if (this._filter.filterRoomTimeline([event]).length) {
+                timelineSet.aggregateRelations(event);
+            }
+        } else {
+            timelineSet.aggregateRelations(event);
+        }
+    }
+};
 
 /**
  * Deal with the echo of a message we sent.
@@ -1204,7 +1279,7 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
  * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
  */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
-    console.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+    logger.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
 
     // if the message was sent, we expect an event id
     if (newStatus == EventStatus.SENT && !newEventId) {
@@ -1236,7 +1311,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
                         newStatus);
     }
 
-    event.status = newStatus;
+    event.setStatus(newStatus);
 
     if (newStatus == EventStatus.SENT) {
         // update the event id
@@ -1251,12 +1326,13 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
     } else if (newStatus == EventStatus.CANCELLED) {
         // remove it from the pending event list, or the timeline.
         if (this._pendingEventList) {
-            utils.removeElement(
+            const idx = this._pendingEventList.findIndex(ev => ev.getId() === oldEventId);
-                this._pendingEventList,
+            if (idx !== -1) {
-                function(ev) {
+                const [removedEvent] = this._pendingEventList.splice(idx, 1);
-                    return ev.getId() == oldEventId;
+                if (removedEvent.getType() === "m.room.redaction") {
-                }, false,
+                    this._revertRedactionLocalEcho(removedEvent);
-            );
+                }
+            }
         }
         this.removeEvent(oldEventId);
     }
@@ -1264,6 +1340,23 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
     this.emit("Room.localEchoUpdated", event, this, oldEventId, oldStatus);
 };
 
+Room.prototype._revertRedactionLocalEcho = function(redactionEvent) {
+    const redactId = redactionEvent.event.redacts;
+    if (!redactId) {
+        return;
+    }
+    const redactedEvent = this.getUnfilteredTimelineSet()
+        .findEventById(redactId);
+    if (redactedEvent) {
+        redactedEvent.unmarkLocallyRedacted();
+        // re-render after undoing redaction
+        this.emit("Room.redactionCancelled", redactionEvent, this);
+        // reapply relation now redaction failed
+        if (redactedEvent.isRelation()) {
+            this._aggregateNonLiveRelation(redactedEvent);
+        }
+    }
+};
 
 /**
  * Add some events to this room. This can include state events, message
@@ -1342,6 +1435,9 @@ Room.prototype.removeEvent = function(eventId) {
     for (let i = 0; i < this._timelineSets.length; i++) {
         const removed = this._timelineSets[i].removeEvent(eventId);
         if (removed) {
+            if (removed.getType() === "m.room.redaction") {
+                this._revertRedactionLocalEcho(removed);
+            }
             removedAny = true;
         }
     }
@@ -1765,10 +1861,21 @@ module.exports = Room;
  * event).
  *
  * @event module:client~MatrixClient#"Room.redaction"
- * @param {MatrixEvent} event The matrix event which was redacted
+ * @param {MatrixEvent} event The matrix redaction event
  * @param {Room} room The room containing the redacted event
  */
 
+/**
+ * Fires when an event that was previously redacted isn't anymore.
+ * This happens when the redaction couldn't be sent and
+ * was subsequently cancelled by the user. Redactions have a local echo
+ * which is undone in this scenario.
+ *
+ * @event module:client~MatrixClient#"Room.redactionCancelled"
+ * @param {MatrixEvent} event The matrix redaction event that was cancelled.
+ * @param {Room} room The room containing the unredacted event
+ */
+
 /**
  * Fires whenever the name of a room is updated.
  * @event module:client~MatrixClient#"Room.name"

src/pushprocessor.js

@@ -23,6 +23,37 @@ import {escapeRegExp, globToRegexp} from "./utils";
 
 const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
 
+// The default override rules to apply when calculating actions for an event. These
+// defaults apply under no other circumstances to avoid confusing the client with server
+// state. We do this for two reasons:
+//   1. Synapse is unlikely to send us the push rule in an incremental sync - see
+//      https://github.com/matrix-org/synapse/pull/4867#issuecomment-481446072 for
+//      more details.
+//   2. We often want to start using push rules ahead of the server supporting them,
+//      and so we can put them here.
+const DEFAULT_OVERRIDE_RULES = [
+    {
+        // For homeservers which don't support MSC1930 yet
+        rule_id: ".m.rule.tombstone",
+        default: true,
+        enabled: true,
+        conditions: [
+            {
+                kind: "event_match",
+                key: "type",
+                pattern: "m.room.tombstone",
+            },
+        ],
+        actions: [
+            "notify",
+            {
+                set_tweak: "highlight",
+                value: true,
+            },
+        ],
+    },
+];
+
 /**
  * Construct a Push Processor.
  * @constructor
@@ -312,6 +343,33 @@ function PushProcessor(client) {
         return actionObj;
     };
 
+    const applyRuleDefaults = function(clientRuleset) {
+        // Deep clone the object before we mutate it
+        const ruleset = JSON.parse(JSON.stringify(clientRuleset));
+
+        if (!clientRuleset['global']) {
+            clientRuleset['global'] = {};
+        }
+        if (!clientRuleset['global']['override']) {
+            clientRuleset['global']['override'] = [];
+        }
+
+        // Apply default overrides
+        const globalOverrides = clientRuleset['global']['override'];
+        for (const override of DEFAULT_OVERRIDE_RULES) {
+            const existingRule = globalOverrides
+                .find((r) => r.rule_id === override.rule_id);
+
+            if (!existingRule) {
+                const ruleId = override.rule_id;
+                console.warn(`Adding default global override for ${ruleId}`);
+                globalOverrides.push(override);
+            }
+        }
+
+        return ruleset;
+    };
+
     this.ruleMatchesEvent = function(rule, ev) {
         let ret = true;
         for (let i = 0; i < rule.conditions.length; ++i) {
@@ -331,7 +389,8 @@ function PushProcessor(client) {
      * @return {PushAction}
      */
     this.actionsForEvent = function(ev) {
-        return pushActionsForEventAndRulesets(ev, client.pushRules);
+        const rules = applyRuleDefaults(client.pushRules);
+        return pushActionsForEventAndRulesets(ev, rules);
     };
 
     /**

src/realtime-callbacks.js

@@ -24,6 +24,7 @@ limitations under the License.
  */
 
 "use strict";
+import logger from '../src/logger';
 
 // we schedule a callback at least this often, to check if we've missed out on
 // some wall-clock time due to being suspended.
@@ -39,7 +40,7 @@ let _realCallbackKey;
 // each is an object with keys [runAt, func, params, key].
 const _callbackList = [];
 
-// var debuglog = console.log.bind(console);
+// var debuglog = logger.log.bind(logger);
 const debuglog = function() {};
 
 /**
@@ -170,7 +171,7 @@ function _runCallbacks() {
         try {
             cb.func.apply(global, cb.params);
         } catch (e) {
-            console.error("Uncaught exception in callback function",
+            logger.error("Uncaught exception in callback function",
                           e.stack || e);
         }
     }

src/scheduler.js

@@ -21,6 +21,7 @@ limitations under the License.
  */
 const utils = require("./utils");
 import Promise from 'bluebird';
+import logger from '../src/logger';
 
 const DEBUG = false;  // set true to enable console logging.
 
@@ -269,7 +270,7 @@ function _removeNextEvent(scheduler, queueName) {
 
 function debuglog() {
     if (DEBUG) {
-        console.log(...arguments);
+        logger.log(...arguments);
     }
 }
 

src/store/indexeddb-local-backend.js

@@ -19,6 +19,7 @@ import Promise from 'bluebird';
 import SyncAccumulator from "../sync-accumulator";
 import utils from "../utils";
 import * as IndexedDBHelpers from "../indexeddb-helpers";
+import logger from '../../src/logger';
 
 const VERSION = 3;
 
@@ -146,7 +147,7 @@ LocalIndexedDBStoreBackend.prototype = {
      */
     connect: function() {
         if (!this._disconnected) {
-            console.log(
+            logger.log(
                 `LocalIndexedDBStoreBackend.connect: already connected or connecting`,
             );
             return Promise.resolve();
@@ -154,14 +155,14 @@ LocalIndexedDBStoreBackend.prototype = {
 
         this._disconnected = false;
 
-        console.log(
+        logger.log(
             `LocalIndexedDBStoreBackend.connect: connecting...`,
         );
         const req = this.indexedDB.open(this._dbName, VERSION);
         req.onupgradeneeded = (ev) => {
             const db = ev.target.result;
             const oldVersion = ev.oldVersion;
-            console.log(
+            logger.log(
                 `LocalIndexedDBStoreBackend.connect: upgrading from ${oldVersion}`,
             );
             if (oldVersion < 1) { // The database did not previously exist.
@@ -178,16 +179,16 @@ LocalIndexedDBStoreBackend.prototype = {
         };
 
         req.onblocked = () => {
-            console.log(
+            logger.log(
                 `can't yet open LocalIndexedDBStoreBackend because it is open elsewhere`,
             );
         };
 
-        console.log(
+        logger.log(
             `LocalIndexedDBStoreBackend.connect: awaiting connection...`,
         );
         return reqAsEventPromise(req).then((ev) => {
-            console.log(
+            logger.log(
                 `LocalIndexedDBStoreBackend.connect: connected`,
             );
             this.db = ev.target.result;
@@ -215,7 +216,7 @@ LocalIndexedDBStoreBackend.prototype = {
             this._loadAccountData(),
             this._loadSyncData(),
         ]).then(([accountData, syncData]) => {
-            console.log(
+            logger.log(
                 `LocalIndexedDBStoreBackend: loaded initial data`,
             );
             this._syncAccumulator.accumulate({
@@ -273,7 +274,7 @@ LocalIndexedDBStoreBackend.prototype = {
                 reject(err);
             };
         }).then((events) => {
-            console.log(`LL: got ${events && events.length}` +
+            logger.log(`LL: got ${events && events.length}` +
                 ` membershipEvents from storage for room ${roomId} ...`);
             return events;
         });
@@ -287,7 +288,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @param {event[]} membershipEvents the membership events to store
      */
     setOutOfBandMembers: async function(roomId, membershipEvents) {
-        console.log(`LL: backend about to store ${membershipEvents.length}` +
+        logger.log(`LL: backend about to store ${membershipEvents.length}` +
             ` members for ${roomId}`);
         const tx = this.db.transaction(["oob_membership_events"], "readwrite");
         const store = tx.objectStore("oob_membership_events");
@@ -306,7 +307,7 @@ LocalIndexedDBStoreBackend.prototype = {
         };
         store.put(markerObject);
         await txnAsPromise(tx);
-        console.log(`LL: backend done storing for ${roomId}!`);
+        logger.log(`LL: backend done storing for ${roomId}!`);
     },
 
     clearOutOfBandMembers: async function(roomId) {
@@ -341,7 +342,7 @@ LocalIndexedDBStoreBackend.prototype = {
             [roomId, maxStateKey],
         );
 
-        console.log(`LL: Deleting all users + marker in storage for ` +
+        logger.log(`LL: Deleting all users + marker in storage for ` +
             `room ${roomId}, with key range:`,
             [roomId, minStateKey], [roomId, maxStateKey]);
         await reqAsPromise(writeStore.delete(membersKeyRange));
@@ -354,11 +355,11 @@ LocalIndexedDBStoreBackend.prototype = {
      */
     clearDatabase: function() {
         return new Promise((resolve, reject) => {
-            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            logger.log(`Removing indexeddb instance: ${this._dbName}`);
             const req = this.indexedDB.deleteDatabase(this._dbName);
 
             req.onblocked = () => {
-                console.log(
+                logger.log(
                     `can't yet delete indexeddb ${this._dbName}` +
                     ` because it is open elsewhere`,
                 );
@@ -368,14 +369,14 @@ LocalIndexedDBStoreBackend.prototype = {
                 // in firefox, with indexedDB disabled, this fails with a
                 // DOMError. We treat this as non-fatal, so that we can still
                 // use the app.
-                console.warn(
+                logger.warn(
                     `unable to delete js-sdk store indexeddb: ${ev.target.error}`,
                 );
                 resolve();
             };
 
             req.onsuccess = () => {
-                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                logger.log(`Removed indexeddb instance: ${this._dbName}`);
                 resolve();
             };
         });
@@ -434,7 +435,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if the data was persisted.
      */
     _persistSyncData: function(nextBatch, roomsData, groupsData) {
-        console.log("Persisting sync data up to ", nextBatch);
+        logger.log("Persisting sync data up to ", nextBatch);
         return Promise.try(() => {
             const txn = this.db.transaction(["sync"], "readwrite");
             const store = txn.objectStore("sync");
@@ -508,7 +509,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise<Object[]>} A list of raw global account events.
      */
     _loadAccountData: function() {
-        console.log(
+        logger.log(
             `LocalIndexedDBStoreBackend: loading account data...`,
         );
         return Promise.try(() => {
@@ -517,7 +518,7 @@ LocalIndexedDBStoreBackend.prototype = {
             return selectQuery(store, undefined, (cursor) => {
                 return cursor.value;
             }).then((result) => {
-                console.log(
+                logger.log(
                     `LocalIndexedDBStoreBackend: loaded account data`,
                 );
                 return result;
@@ -530,7 +531,7 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
      */
     _loadSyncData: function() {
-        console.log(
+        logger.log(
             `LocalIndexedDBStoreBackend: loading sync data...`,
         );
         return Promise.try(() => {
@@ -539,11 +540,11 @@ LocalIndexedDBStoreBackend.prototype = {
             return selectQuery(store, undefined, (cursor) => {
                 return cursor.value;
             }).then((results) => {
-                console.log(
+                logger.log(
                     `LocalIndexedDBStoreBackend: loaded sync data`,
                 );
                 if (results.length > 1) {
-                    console.warn("loadSyncData: More than 1 sync row found.");
+                    logger.warn("loadSyncData: More than 1 sync row found.");
                 }
                 return (results.length > 0 ? results[0] : {});
             });

src/store/indexeddb-remote-backend.js

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 import Promise from 'bluebird';
+import logger from '../../src/logger';
 
 /**
  * An IndexedDB store backend where the actual backend sits in a web
@@ -140,7 +141,7 @@ RemoteIndexedDBStoreBackend.prototype = {
 
             // tell the worker the db name.
             this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
-                console.log("IndexedDB worker is ready");
+                logger.log("IndexedDB worker is ready");
             });
         }
         return this._startPromise;
@@ -170,13 +171,13 @@ RemoteIndexedDBStoreBackend.prototype = {
 
         if (msg.command == 'cmd_success' || msg.command == 'cmd_fail') {
             if (msg.seq === undefined) {
-                console.error("Got reply from worker with no seq");
+                logger.error("Got reply from worker with no seq");
                 return;
             }
 
             const def = this._inFlight[msg.seq];
             if (def === undefined) {
-                console.error("Got reply for unknown seq " + msg.seq);
+                logger.error("Got reply for unknown seq " + msg.seq);
                 return;
             }
             delete this._inFlight[msg.seq];
@@ -189,7 +190,7 @@ RemoteIndexedDBStoreBackend.prototype = {
                 def.reject(error);
             }
         } else {
-            console.warn("Unrecognised message from worker: " + msg);
+            logger.warn("Unrecognised message from worker: " + msg);
         }
     },
 };

src/store/indexeddb-store-worker.js

@@ -17,6 +17,7 @@ limitations under the License.
 
 import Promise from 'bluebird';
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+import logger from '../../src/logger';
 
 /**
  * This class lives in the webworker and drives a LocalIndexedDBStoreBackend
@@ -129,8 +130,8 @@ class IndexedDBStoreWorker {
                 result: ret,
             });
         }, (err) => {
-            console.error("Error running command: "+msg.command);
+            logger.error("Error running command: "+msg.command);
-            console.error(err);
+            logger.error(err);
             this.postMessage.call(null, {
                 command: 'cmd_fail',
                 seq: msg.seq,

src/store/indexeddb.js

@@ -15,13 +15,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+/* eslint-disable babel/no-invalid-this */
+
 import Promise from 'bluebird';
 import {MemoryStore} from "./memory";
 import utils from "../utils";
+import {EventEmitter} from 'events';
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
 import RemoteIndexedDBStoreBackend from "./indexeddb-remote-backend.js";
 import User from "../models/user";
 import {MatrixEvent} from "../models/event";
+import logger from '../../src/logger';
 
 /**
  * This is an internal module. See {@link IndexedDBStore} for the public class.
@@ -110,6 +114,7 @@ const IndexedDBStore = function IndexedDBStore(opts) {
     };
 };
 utils.inherits(IndexedDBStore, MemoryStore);
+utils.extend(IndexedDBStore.prototype, EventEmitter.prototype);
 
 IndexedDBStore.exists = function(indexedDB, dbName) {
     return LocalIndexedDBStoreBackend.exists(indexedDB, dbName);
@@ -120,16 +125,16 @@ IndexedDBStore.exists = function(indexedDB, dbName) {
   */
 IndexedDBStore.prototype.startup = function() {
     if (this.startedUp) {
-        console.log(`IndexedDBStore.startup: already started`);
+        logger.log(`IndexedDBStore.startup: already started`);
         return Promise.resolve();
     }
 
-    console.log(`IndexedDBStore.startup: connecting to backend`);
+    logger.log(`IndexedDBStore.startup: connecting to backend`);
     return this.backend.connect().then(() => {
-        console.log(`IndexedDBStore.startup: loading presence events`);
+        logger.log(`IndexedDBStore.startup: loading presence events`);
         return this.backend.getUserPresenceEvents();
     }).then((userPresenceEvents) => {
-        console.log(`IndexedDBStore.startup: processing presence events`);
+        logger.log(`IndexedDBStore.startup: processing presence events`);
         userPresenceEvents.forEach(([userId, rawEvent]) => {
             const u = new User(userId);
             if (rawEvent) {
@@ -146,36 +151,36 @@ IndexedDBStore.prototype.startup = function() {
  * client state to where it was at the last save, or null if there
  * is no saved sync data.
  */
-IndexedDBStore.prototype.getSavedSync = function() {
+IndexedDBStore.prototype.getSavedSync = degradable(function() {
     return this.backend.getSavedSync();
-};
+}, "getSavedSync");
 
 /** @return {Promise<bool>} whether or not the database was newly created in this session. */
-IndexedDBStore.prototype.isNewlyCreated = function() {
+IndexedDBStore.prototype.isNewlyCreated = degradable(function() {
     return this.backend.isNewlyCreated();
-};
+}, "isNewlyCreated");
 
 /**
  * @return {Promise} If there is a saved sync, the nextBatch token
  * for this sync, otherwise null.
  */
-IndexedDBStore.prototype.getSavedSyncToken = function() {
+IndexedDBStore.prototype.getSavedSyncToken = degradable(function() {
     return this.backend.getNextBatchToken();
-},
+}, "getSavedSyncToken"),
 
 /**
  * Delete all data from this store.
  * @return {Promise} Resolves if the data was deleted from the database.
  */
-IndexedDBStore.prototype.deleteAllData = function() {
+IndexedDBStore.prototype.deleteAllData = degradable(function() {
     MemoryStore.prototype.deleteAllData.call(this);
     return this.backend.clearDatabase().then(() => {
-        console.log("Deleted indexeddb data.");
+        logger.log("Deleted indexeddb data.");
     }, (err) => {
-        console.error(`Failed to delete indexeddb data: ${err}`);
+        logger.error(`Failed to delete indexeddb data: ${err}`);
         throw err;
     });
-};
+});
 
 /**
  * Whether this store would like to save its data
@@ -203,7 +208,7 @@ IndexedDBStore.prototype.save = function() {
     return Promise.resolve();
 };
 
-IndexedDBStore.prototype._reallySave = function() {
+IndexedDBStore.prototype._reallySave = degradable(function() {
     this._syncTs = Date.now(); // set now to guard against multi-writes
 
     // work out changed users (this doesn't handle deletions but you
@@ -219,14 +224,12 @@ IndexedDBStore.prototype._reallySave = function() {
         this._userModifiedMap[u.userId] = u.getLastModifiedTime();
     }
 
-    return this.backend.syncToDatabase(userTuples).catch((err) => {
+    return this.backend.syncToDatabase(userTuples);
-        console.error("sync fail:", err);
+});
-    });
-};
 
-IndexedDBStore.prototype.setSyncData = function(syncData) {
+IndexedDBStore.prototype.setSyncData = degradable(function(syncData) {
     return this.backend.setSyncData(syncData);
-};
+}, "setSyncData");
 
 /**
  * Returns the out-of-band membership events for this room that
@@ -235,9 +238,9 @@ IndexedDBStore.prototype.setSyncData = function(syncData) {
  * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
  * @returns {null} in case the members for this room haven't been stored yet
  */
-IndexedDBStore.prototype.getOutOfBandMembers = function(roomId) {
+IndexedDBStore.prototype.getOutOfBandMembers = degradable(function(roomId) {
     return this.backend.getOutOfBandMembers(roomId);
-};
+}, "getOutOfBandMembers");
 
 /**
  * Stores the out-of-band membership events for this room. Note that
@@ -247,20 +250,71 @@ IndexedDBStore.prototype.getOutOfBandMembers = function(roomId) {
  * @param {event[]} membershipEvents the membership events to store
  * @returns {Promise} when all members have been stored
  */
-IndexedDBStore.prototype.setOutOfBandMembers = function(roomId, membershipEvents) {
+IndexedDBStore.prototype.setOutOfBandMembers = degradable(function(
+    roomId,
+    membershipEvents,
+) {
+    MemoryStore.prototype.setOutOfBandMembers.call(this, roomId, membershipEvents);
     return this.backend.setOutOfBandMembers(roomId, membershipEvents);
-};
+}, "setOutOfBandMembers");
 
-IndexedDBStore.prototype.clearOutOfBandMembers = function(roomId) {
+IndexedDBStore.prototype.clearOutOfBandMembers = degradable(function(roomId) {
+    MemoryStore.prototype.clearOutOfBandMembers.call(this);
     return this.backend.clearOutOfBandMembers(roomId);
-};
+}, "clearOutOfBandMembers");
 
-IndexedDBStore.prototype.getClientOptions = function() {
+IndexedDBStore.prototype.getClientOptions = degradable(function() {
     return this.backend.getClientOptions();
-};
+}, "getClientOptions");
 
-IndexedDBStore.prototype.storeClientOptions = function(options) {
+IndexedDBStore.prototype.storeClientOptions = degradable(function(options) {
+    MemoryStore.prototype.storeClientOptions.call(this, options);
     return this.backend.storeClientOptions(options);
-};
+}, "storeClientOptions");
 
 module.exports.IndexedDBStore = IndexedDBStore;
+
+/**
+ * All member functions of `IndexedDBStore` that access the backend use this wrapper to
+ * watch for failures after initial store startup, including `QuotaExceededError` as
+ * free disk space changes, etc.
+ *
+ * When IndexedDB fails via any of these paths, we degrade this back to a `MemoryStore`
+ * in place so that the current operation and all future ones are in-memory only.
+ *
+ * @param {Function} func The degradable work to do.
+ * @param {String} fallback The method name for fallback.
+ * @returns {Function} A wrapped member function.
+ */
+function degradable(func, fallback) {
+    return async function(...args) {
+        try {
+            return await func.call(this, ...args);
+        } catch (e) {
+            logger.error("IndexedDBStore failure, degrading to MemoryStore", e);
+            this.emit("degraded", e);
+            try {
+                // We try to delete IndexedDB after degrading since this store is only a
+                // cache (the app will still function correctly without the data).
+                // It's possible that deleting repair IndexedDB for the next app load,
+                // potenially by making a little more space available.
+                logger.log("IndexedDBStore trying to delete degraded data");
+                await this.backend.clearDatabase();
+                logger.log("IndexedDBStore delete after degrading succeeeded");
+            } catch (e) {
+                logger.warn("IndexedDBStore delete after degrading failed", e);
+            }
+            // Degrade the store from being an instance of `IndexedDBStore` to instead be
+            // an instance of `MemoryStore` so that future API calls use the memory path
+            // directly and skip IndexedDB entirely. This should be safe as
+            // `IndexedDBStore` already extends from `MemoryStore`, so we are making the
+            // store become its parent type in a way. The mutator methods of
+            // `IndexedDBStore` also maintain the state that `MemoryStore` uses (many are
+            // not overridden at all).
+            Object.setPrototypeOf(this, MemoryStore.prototype);
+            if (fallback) {
+                return await MemoryStore.prototype[fallback].call(this, ...args);
+            }
+        }
+    };
+}

src/store/memory.js

@@ -385,6 +385,7 @@ module.exports.MemoryStore.prototype = {
         };
         return Promise.resolve();
     },
+
     /**
      * Returns the out-of-band membership events for this room that
      * were previously loaded.
@@ -395,6 +396,7 @@ module.exports.MemoryStore.prototype = {
     getOutOfBandMembers: function(roomId) {
         return Promise.resolve(this._oobMembers[roomId] || null);
     },
+
     /**
      * Stores the out-of-band membership events for this room. Note that
      * it still makes sense to store an empty array as the OOB status for the room is
@@ -408,6 +410,11 @@ module.exports.MemoryStore.prototype = {
         return Promise.resolve();
     },
 
+    clearOutOfBandMembers: function() {
+        this._oobMembers = {};
+        return Promise.resolve();
+    },
+
     getClientOptions: function() {
         return Promise.resolve(this._clientOptions);
     },

src/store/session/webstorage.js

@@ -22,6 +22,7 @@ limitations under the License.
  */
 
 const utils = require("../../utils");
+import logger from '../../logger';
 
 const DEBUG = false;  // set true to enable console logging.
 const E2E_PREFIX = "session.e2e.";
@@ -257,7 +258,7 @@ function removeByPrefix(store, prefix) {
 
 function debuglog() {
     if (DEBUG) {
-        console.log(...arguments);
+        logger.log(...arguments);
     }
 }
 

src/sync-accumulator.js

@@ -21,6 +21,7 @@ limitations under the License.
  */
 
 import utils from "./utils";
+import logger from '../src/logger';
 
 
 /**
@@ -168,7 +169,7 @@ class SyncAccumulator {
                 }
                 break;
             default:
-                console.error("Unknown cateogory: ", category);
+                logger.error("Unknown cateogory: ", category);
         }
     }
 

src/sync.js

@@ -32,6 +32,7 @@ const Group = require('./models/group');
 const utils = require("./utils");
 const Filter = require("./filter");
 const EventTimeline = require("./models/event-timeline");
+import logger from '../src/logger';
 
 import {InvalidStoreError} from './errors';
 
@@ -58,7 +59,7 @@ function debuglog(...params) {
     if (!DEBUG) {
         return;
     }
-    console.log(...params);
+    logger.log(...params);
 }
 
 
@@ -116,17 +117,25 @@ function SyncApi(client, opts) {
  */
 SyncApi.prototype.createRoom = function(roomId) {
     const client = this.client;
+    const {
+        timelineSupport,
+        unstableClientRelationAggregation,
+    } = client;
     const room = new Room(roomId, client, client.getUserId(), {
         lazyLoadMembers: this.opts.lazyLoadMembers,
         pendingEventOrdering: this.opts.pendingEventOrdering,
-        timelineSupport: client.timelineSupport,
+        timelineSupport,
+        unstableClientRelationAggregation,
     });
-    client.reEmitter.reEmit(room, ["Room.name", "Room.timeline", "Room.redaction",
+    client.reEmitter.reEmit(room, ["Room.name", "Room.timeline",
+                          "Room.redaction",
+                          "Room.redactionCancelled",
                           "Room.receipt", "Room.tags",
                           "Room.timelineReset",
                           "Room.localEchoUpdated",
                           "Room.accountData",
                           "Room.myMembership",
+                          "Room.replaceEvent",
                          ]);
     this._registerStateListeners(room);
     return room;
@@ -386,7 +395,7 @@ SyncApi.prototype._peekPoll = function(peekRoom, token) {
         peekRoom.addLiveEvents(events);
         self._peekPoll(peekRoom, res.end);
     }, function(err) {
-        console.error("[%s] Peek poll failed: %s", peekRoom.roomId, err);
+        logger.error("[%s] Peek poll failed: %s", peekRoom.roomId, err);
         setTimeout(function() {
             self._peekPoll(peekRoom, token);
         }, 30 * 1000);
@@ -445,6 +454,16 @@ SyncApi.prototype._wasLazyLoadingToggled = async function(lazyLoadMembers) {
     return false;
 };
 
+SyncApi.prototype._shouldAbortSync = function(error) {
+    if (error.errcode === "M_UNKNOWN_TOKEN") {
+        // The logout already happened, we just need to stop.
+        logger.warn("Token no longer valid - assuming logout");
+        this.stop();
+        return true;
+    }
+    return false;
+};
+
 /**
  * Main entry point
  */
@@ -472,13 +491,17 @@ SyncApi.prototype.sync = function() {
 
     async function getPushRules() {
         try {
+            debuglog("Getting push rules...");
             const result = await client.getPushRules();
             debuglog("Got push rules");
 
             client.pushRules = result;
         } catch (err) {
+            logger.error("Getting push rules failed", err);
+            if (self._shouldAbortSync(err)) return;
             // wait for saved sync to complete before doing anything else,
             // otherwise the sync state will end up being incorrect
+            debuglog("Waiting for saved sync before retrying push rules...");
             await self.recoverFromSyncStartupError(savedSyncPromise, err);
             getPushRules();
             return;
@@ -487,22 +510,35 @@ SyncApi.prototype.sync = function() {
     }
 
     const checkLazyLoadStatus = async () => {
+        debuglog("Checking lazy load status...");
         if (this.opts.lazyLoadMembers && client.isGuest()) {
             this.opts.lazyLoadMembers = false;
         }
         if (this.opts.lazyLoadMembers) {
+            debuglog("Checking server lazy load support...");
             const supported = await client.doesServerSupportLazyLoading();
             if (supported) {
+                try {
+                    debuglog("Creating and storing lazy load sync filter...");
                     this.opts.filter = await client.createFilter(
                         Filter.LAZY_LOADING_SYNC_FILTER,
                     );
+                    debuglog("Created and stored lazy load sync filter");
+                } catch (err) {
+                    logger.error(
+                        "Creating and storing lazy load sync filter failed",
+                        err,
+                    );
+                    throw err;
+                }
             } else {
-                console.log("LL: lazy loading requested but not supported " +
+                debuglog("LL: lazy loading requested but not supported " +
                     "by server, so disabling");
                 this.opts.lazyLoadMembers = false;
             }
         }
         // need to vape the store when enabling LL and wasn't enabled before
+        debuglog("Checking whether lazy loading has changed in store...");
         const shouldClear = await this._wasLazyLoadingToggled(this.opts.lazyLoadMembers);
         if (shouldClear) {
             this._storeIsInvalid = true;
@@ -513,18 +549,26 @@ SyncApi.prototype.sync = function() {
             // we leave the state as 'ERROR' which isn't great since this normally means
             // we're retrying. The client must be stopped before clearing the stores anyway
             // so the app should stop the client, clear the store and start it again.
-            console.warn("InvalidStoreError: store is not usable: stopping sync.");
+            logger.warn("InvalidStoreError: store is not usable: stopping sync.");
             return;
         }
         if (this.opts.lazyLoadMembers && this.opts.crypto) {
             this.opts.crypto.enableLazyLoading();
         }
+        try {
+            debuglog("Storing client options...");
             await this.client._storeClientOptions();
+            debuglog("Stored client options");
+        } catch (err) {
+            logger.error("Storing client options failed", err);
+            throw err;
+        }
 
         getFilter(); // Now get the filter and start syncing
     };
 
     async function getFilter() {
+        debuglog("Getting filter...");
         let filter;
         if (self.opts.filter) {
             filter = self.opts.filter;
@@ -539,8 +583,11 @@ SyncApi.prototype.sync = function() {
                 getFilterName(client.credentials.userId), filter,
             );
         } catch (err) {
+            logger.error("Getting filter failed", err);
+            if (self._shouldAbortSync(err)) return;
             // wait for saved sync to complete before doing anything else,
             // otherwise the sync state will end up being incorrect
+            debuglog("Waiting for saved sync before retrying filter...");
             await self.recoverFromSyncStartupError(savedSyncPromise, err);
             getFilter();
             return;
@@ -554,11 +601,12 @@ SyncApi.prototype.sync = function() {
         if (self._currentSyncRequest === null) {
             // Send this first sync request here so we can then wait for the saved
             // sync data to finish processing before we process the results of this one.
-            console.log("Sending first sync request...");
+            debuglog("Sending first sync request...");
             self._currentSyncRequest = self._doSyncRequest({ filterId }, savedSyncToken);
         }
 
         // Now wait for the saved sync to finish...
+        debuglog("Waiting for saved sync before starting sync processing...");
         await savedSyncPromise;
         self._sync({ filterId });
     }
@@ -570,13 +618,19 @@ SyncApi.prototype.sync = function() {
         // Pull the saved sync token out first, before the worker starts sending
         // all the sync data which could take a while. This will let us send our
         // first incremental sync request before we've processed our saved data.
+        debuglog("Getting saved sync token...");
         savedSyncPromise = client.store.getSavedSyncToken().then((tok) => {
+            debuglog("Got saved sync token");
             savedSyncToken = tok;
+            debuglog("Getting saved sync...");
             return client.store.getSavedSync();
         }).then((savedSync) => {
+            debuglog(`Got reply from saved sync, exists? ${!!savedSync}`);
             if (savedSync) {
                 return self._syncFromCache(savedSync);
             }
+        }).catch(err => {
+            logger.error("Getting saved sync failed", err);
         });
         // Now start the first incremental sync request: this can also
         // take a while so if we set it going now, we can wait for it
@@ -648,7 +702,7 @@ SyncApi.prototype._syncFromCache = async function(savedSync) {
     try {
         await this._processSyncResponse(syncEventData, data);
     } catch(e) {
-        console.error("Error processing cached sync", e.stack || e);
+        logger.error("Error processing cached sync", e.stack || e);
     }
 
     // Don't emit a prepared if we've bailed because the store is invalid:
@@ -723,7 +777,7 @@ SyncApi.prototype._sync = async function(syncOptions) {
     } catch(e) {
         // log the exception with stack if we have it, else fall back
         // to the plain description
-        console.error("Caught /sync error", e.stack || e);
+        logger.error("Caught /sync error", e.stack || e);
 
         // Emit the exception for client handling
         this.client.emit("sync.unexpectedError", e);
@@ -837,11 +891,15 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
         return;
     }
 
-    console.error("/sync error %s", err);
+    logger.error("/sync error %s", err);
-    console.error(err);
+    logger.error(err);
+
+    if(this._shouldAbortSync(err)) {
+        return;
+    }
 
     this._failedSyncCount++;
-    console.log('Number of consecutive failed sync requests:', this._failedSyncCount);
+    logger.log('Number of consecutive failed sync requests:', this._failedSyncCount);
 
     debuglog("Starting keep-alive");
     // Note that we do *not* mark the sync connection as
@@ -995,7 +1053,7 @@ SyncApi.prototype._processSyncResponse = async function(
                             content.msgtype == "m.bad.encrypted"
                     ) {
                         // the mapper already logged a warning.
-                        console.log(
+                        logger.log(
                             'Ignoring undecryptable to-device event from ' +
                                 toDeviceEvent.getSender(),
                         );
@@ -1051,7 +1109,6 @@ SyncApi.prototype._processSyncResponse = async function(
         const stateEvents =
             self._mapSyncEventsFormat(inviteObj.invite_state, room);
 
-        room.updateMyMembership("invite");
         self._processRoomEvents(room, stateEvents);
         if (inviteObj.isBrandNewRoom) {
             room.recalculate();
@@ -1061,6 +1118,7 @@ SyncApi.prototype._processSyncResponse = async function(
         stateEvents.forEach(function(e) {
             client.emit("event", e);
         });
+        room.updateMyMembership("invite");
     });
 
     // Handle joins
@@ -1076,12 +1134,19 @@ SyncApi.prototype._processSyncResponse = async function(
             room.setUnreadNotificationCount(
                 'total', joinObj.unread_notifications.notification_count,
             );
+
+            // We track unread notifications ourselves in encrypted rooms, so don't
+            // bother setting it here. We trust our calculations better than the
+            // server's for this case, and therefore will assume that our non-zero
+            // count is accurate.
+            const encrypted = client.isRoomEncrypted(room.roomId);
+            if (!encrypted
+                || (encrypted && room.getUnreadNotificationCount('highlight') <= 0)) {
                 room.setUnreadNotificationCount(
                     'highlight', joinObj.unread_notifications.highlight_count,
                 );
             }
-
+        }
-        room.updateMyMembership("join");
 
         joinObj.timeline = joinObj.timeline || {};
 
@@ -1195,6 +1260,8 @@ SyncApi.prototype._processSyncResponse = async function(
         accountDataEvents.forEach(function(e) {
             client.emit("event", e);
         });
+
+        room.updateMyMembership("join");
     });
 
     // Handle leaves (e.g. kicked rooms)
@@ -1207,8 +1274,6 @@ SyncApi.prototype._processSyncResponse = async function(
         const accountDataEvents =
             self._mapSyncEventsFormat(leaveObj.account_data);
 
-        room.updateMyMembership("leave");
-
         self._processRoomEvents(room, stateEvents, timelineEvents);
         room.addAccountData(accountDataEvents);
 
@@ -1229,6 +1294,8 @@ SyncApi.prototype._processSyncResponse = async function(
         accountDataEvents.forEach(function(e) {
             client.emit("event", e);
         });
+
+        room.updateMyMembership("leave");
     });
 
     // update the notification timeline, if appropriate.

src/timeline-window.js

@@ -19,6 +19,7 @@ limitations under the License.
 
 import Promise from 'bluebird';
 const EventTimeline = require("./models/event-timeline");
+import logger from '../src/logger';
 
 /**
  * @private
@@ -28,7 +29,7 @@ const DEBUG = false;
 /**
  * @private
  */
-const debuglog = DEBUG ? console.log.bind(console) : function() {};
+const debuglog = DEBUG ? logger.log.bind(logger) : function() {};
 
 /**
  * the number of times we ask the server for more events before giving up

src/webrtc/call.js

@@ -21,6 +21,7 @@ limitations under the License.
  */
 const utils = require("../utils");
 const EventEmitter = require("events").EventEmitter;
+import logger from '../../src/logger';
 const DEBUG = true;  // set true to enable console logging.
 
 // events: hangup, error(err), replaced(call), state(state, oldState)
@@ -195,7 +196,7 @@ MatrixCall.prototype.placeScreenSharingCall =
  * @param {string} queueId Arbitrary ID to track the chain of promises to be used
  */
 MatrixCall.prototype.playElement = function(element, queueId) {
-    console.log("queuing play on " + queueId + " and element " + element);
+    logger.log("queuing play on " + queueId + " and element " + element);
     // XXX: FIXME: Does this leak elements, given the old promises
     // may hang around and retain a reference to them?
     if (this.mediaPromises[queueId]) {
@@ -206,10 +207,10 @@ MatrixCall.prototype.playElement = function(element, queueId) {
         // these failures may be non-fatal (as in the case of unmounts)
         this.mediaPromises[queueId] =
             this.mediaPromises[queueId].then(function() {
-                console.log("previous promise completed for " + queueId);
+                logger.log("previous promise completed for " + queueId);
                 return element.play();
             }, function() {
-                console.log("previous promise failed for " + queueId);
+                logger.log("previous promise failed for " + queueId);
                 return element.play();
             });
     } else {
@@ -224,14 +225,14 @@ MatrixCall.prototype.playElement = function(element, queueId) {
  * @param {string} queueId Arbitrary ID to track the chain of promises to be used
  */
 MatrixCall.prototype.pauseElement = function(element, queueId) {
-    console.log("queuing pause on " + queueId + " and element " + element);
+    logger.log("queuing pause on " + queueId + " and element " + element);
     if (this.mediaPromises[queueId]) {
         this.mediaPromises[queueId] =
             this.mediaPromises[queueId].then(function() {
-                console.log("previous promise completed for " + queueId);
+                logger.log("previous promise completed for " + queueId);
                 return element.pause();
             }, function() {
-                console.log("previous promise failed for " + queueId);
+                logger.log("previous promise failed for " + queueId);
                 return element.pause();
             });
     } else {
@@ -250,15 +251,15 @@ MatrixCall.prototype.pauseElement = function(element, queueId) {
  * @param {string} queueId Arbitrary ID to track the chain of promises to be used
  */
 MatrixCall.prototype.assignElement = function(element, srcObject, queueId) {
-    console.log("queuing assign on " + queueId + " element " + element + " for " +
+    logger.log("queuing assign on " + queueId + " element " + element + " for " +
         srcObject);
     if (this.mediaPromises[queueId]) {
         this.mediaPromises[queueId] =
             this.mediaPromises[queueId].then(function() {
-                console.log("previous promise completed for " + queueId);
+                logger.log("previous promise completed for " + queueId);
                 element.srcObject = srcObject;
             }, function() {
-                console.log("previous promise failed for " + queueId);
+                logger.log("previous promise failed for " + queueId);
                 element.srcObject = srcObject;
             });
     } else {
@@ -1159,7 +1160,7 @@ const callError = function(code, msg) {
 
 const debuglog = function() {
     if (DEBUG) {
-        console.log(...arguments);
+        logger.log(...arguments);
     }
 };