Store account data in the same way as room data

Previously, we treated the `MatrixEvents` that were in `this.accountData` in
`MatrixInMemoryStore` as the ground truth and saved those to disk and restored
them back upon load. This did not consider that there are **no emitted events**
when they are restored. Riot-Web was listening for a specific account data
event in order to dynamically update the theme. When the page was reloaded, we
dutifully put the right event in `MatrixInMemoryStore`, but failed to emit an
event to tell Riot-Web this. This led to vector-im/riot-web#3247

This patch fixes it by treating the `/sync` response as the ground truth and
ignoring `this.accountData` entirely. This means that upon load, we will be
injecting an `account_data` key into the initial `/sync` response. This will
cause it to be added to `this.accountData` in the store AND cause the event to
be emitted.

src/store/indexeddb.js

@@ -116,7 +116,7 @@ IndexedDBStoreBackend.prototype = {
     /**
      * Persist a list of account data events. Events with the same 'type' will
      * be replaced.
-     * @param {MatrixEvent[]} accountData An array of user-scoped account data events
+     * @param {Object[]} accountData An array of raw user-scoped account data events
      * @return {Promise} Resolves if the events were persisted.
      */
     persistAccountData: function(accountData) {
@@ -124,7 +124,7 @@ IndexedDBStoreBackend.prototype = {
             const txn = this.db.transaction(["accountData"], "readwrite");
             const store = txn.objectStore("accountData");
             for (let i = 0; i < accountData.length; i++) {
-                store.put(accountData[i].event); // put == UPSERT
+                store.put(accountData[i]); // put == UPSERT
             }
             return promiseifyTxn(txn);
         });
@@ -172,14 +172,14 @@ IndexedDBStoreBackend.prototype = {
 
     /**
      * Load all the account data events from the database. This is not cached.
-     * @return {Promise<MatrixEvent[]>} A list of events.
+     * @return {Promise<Object[]>} A list of raw global account events.
      */
     loadAccountData: function() {
         return q.try(() => {
             const txn = this.db.transaction(["accountData"], "readonly");
             const store = txn.objectStore("accountData");
             return selectQuery(store, undefined, (cursor) => {
-                return new MatrixEvent(cursor.value);
+                return cursor.value;
             });
         });
     },
@@ -283,10 +283,9 @@ IndexedDBStore.prototype.startup = function() {
             this._userModifiedMap[u.userId] = u.getLastModifiedTime();
             this.storeUser(u);
         });
-        this.storeAccountDataEvents(accountData);
         this._syncTs = Date.now(); // pretend we've written so we don't rewrite
         this.setSyncToken(syncData.nextBatch);
-        this._setSyncData(syncData.nextBatch, syncData.roomsData);
+        this._setSyncData(syncData.nextBatch, syncData.roomsData, accountData);
     });
 };
 
@@ -325,10 +324,13 @@ IndexedDBStore.prototype.save = function() {
     return q();
 };
 
-IndexedDBStore.prototype._setSyncData = function(nextBatch, roomsData) {
-    this._syncAccumulator.accumulateRooms({
+IndexedDBStore.prototype._setSyncData = function(nextBatch, roomsData, accountData) {
+    this._syncAccumulator.accumulate({
         next_batch: nextBatch,
         rooms: roomsData,
+        account_data: {
+            events: accountData,
+        },
     });
 };
 
@@ -344,18 +346,11 @@ IndexedDBStore.prototype._syncToDatabase = function() {
         this._userModifiedMap[u.userId] = u.getLastModifiedTime();
     });
 
-    // TODO: work out changed account data events. They don't have timestamps or IDs.
-    // so we'll need to hook into storeAccountDataEvents instead to catch them when
-    // they update from /sync
-    const changedAccountData = Object.keys(this.accountData).map((etype) => {
-        return this.accountData[etype];
-    });
-
     const syncData = this._syncAccumulator.getJSON();
 
     return q.all([
         this.backend.persistUsers(changedUsers),
-        this.backend.persistAccountData(changedAccountData),
+        this.backend.persistAccountData(syncData.accountData),
         this.backend.persistSyncData(syncData.nextBatch, syncData.roomsData),
     ]);
 };

src/sync-accumulator.js

@@ -47,6 +47,9 @@ class SyncAccumulator {
         opts = opts || {};
         opts.maxTimelineEntries = opts.maxTimelineEntries || 50;
         this.opts = opts;
+        this.accountData = {
+            //$event_type: Object
+        };
         this.inviteRooms = {
             //$roomId: { ... sync 'invite' json data ... }
         };
@@ -71,15 +74,30 @@ class SyncAccumulator {
         this.nextBatch = null;
     }
 
+    accumulate(syncResponse) {
+        this._accumulateRooms(syncResponse);
+        this._accumulateAccountData(syncResponse);
+        this.nextBatch = syncResponse.next_batch;
+    }
+
+    _accumulateAccountData(syncResponse) {
+        if (!syncResponse.account_data || !syncResponse.account_data.events) {
+            return;
+        }
+        // Clobbers based on event type.
+        syncResponse.account_data.events.forEach((e) => {
+            this.accountData[e.type] = e;
+        });
+    }
+
     /**
      * Accumulate incremental /sync room data.
      * @param {Object} syncResponse the complete /sync JSON
      */
-    accumulateRooms(syncResponse) {
+    _accumulateRooms(syncResponse) {
         if (!syncResponse.rooms) {
             return;
         }
-        this.nextBatch = syncResponse.next_batch;
         if (syncResponse.rooms.invite) {
             Object.keys(syncResponse.rooms.invite).forEach((roomId) => {
                 this._accumulateRoom(
@@ -316,13 +334,15 @@ class SyncAccumulator {
      * Return everything under the 'rooms' key from a /sync response which
      * represents all room data that should be stored. This should be paired
      * with the sync token which represents the most recent /sync response
-     * provided to accumulateRooms().
-     * @return {Object} An object with a "nextBatch" key and a "roomsData" key.
+     * provided to accumulate().
+     * @return {Object} An object with a "nextBatch", "roomsData" and "accountData"
+     * keys.
      * The "nextBatch" key is a string which represents at what point in the
      * /sync stream the accumulator reached. This token should be used when
      * restarting a /sync stream at startup. Failure to do so can lead to missing
      * events. The "roomsData" key is an Object which represents the entire
-     * /sync response from the 'rooms' key onwards.
+     * /sync response from the 'rooms' key onwards. The "accountData" key is
+     * a list of raw events which represent global account data.
      */
     getJSON() {
         const data = {
@@ -434,9 +454,17 @@ class SyncAccumulator {
             });
             data.join[roomId] = roomJson;
         });
+
+        // Add account data
+        const accData = [];
+        Object.keys(this.accountData).forEach((evType) => {
+            accData.push(this.accountData[evType]);
+        });
+
         return {
             nextBatch: this.nextBatch,
             roomsData: data,
+            accountData: accData,
         };
     }
 }

src/sync.js

@@ -544,6 +544,9 @@ SyncApi.prototype._sync = function(syncOptions) {
             this._currentSyncRequest = q.resolve({
                 next_batch: data.nextBatch,
                 rooms: data.roomsData,
+                account_data: {
+                    events: data.accountData,
+                },
             });
             isCachedResponse = true;
         }
@@ -576,7 +579,7 @@ SyncApi.prototype._sync = function(syncOptions) {
         // accumulated data. We don't want to accumulate the same thing twice, so
         // only accumulate if this isn't a cached response.
         if (self.opts.syncAccumulator && !isCachedResponse) {
-            self.opts.syncAccumulator.accumulateRooms(data);
+            self.opts.syncAccumulator.accumulate(data);
         }
 
         // emit synced events