Add ability to do indexeddb sync work in webworker

src/matrix.js

@@ -61,6 +61,8 @@ module.exports.Filter = require("./filter");
 module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
 /** The {@link module:interactive-auth} class. */
 module.exports.InteractiveAuth = require("./interactive-auth");
+/** The {@link module:indexeddb-remote-worker} class. */
+module.exports.IndexedDbStoreWorker = require("./store/indexeddb-remote-worker.js");
 
 
 /**

src/store/indexeddb-local-backend.js

@@ -15,8 +15,6 @@ limitations under the License.
 */
 
 import q from "q";
-import User from "../models/user";
-import {MatrixEvent} from "../models/event";
 import SyncAccumulator from "../sync-accumulator";
 import utils from "../utils";
 
@@ -104,13 +102,6 @@ const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
     this._dbName = "matrix-js-sdk:" + (dbName || "default");
     this.db = null;
     this._syncAccumulator = new SyncAccumulator();
-
-    // Records the last-modified-time of each user at the last point we saved
-    // the database, such that we can derive the set if users that have been
-    // modified since we last saved.
-    this._userModifiedMap = {
-        // user_id : timestamp
-    };
 };
 
 
@@ -179,23 +170,16 @@ LocalIndexedDBStoreBackend.prototype = {
     },
 
     setSyncData: function(syncData) {
-        this._syncAccumulator.accumulate(syncData);
+        return q().then(() => {
+            this._syncAccumulator.accumulate(syncData);
+        });
     },
 
-    syncToDatabase: function(users) {
+    syncToDatabase: function(userTuples) {
-        // work out changed users (this doesn't handle deletions but you
-        // can't 'delete' users as they are just presence events).
-        const changedUsers = users.filter((user) => {
-            return this._userModifiedMap[user.userId] !== user.getLastModifiedTime();
-        });
-        changedUsers.forEach((u) => { // update times
-            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
-        });
-
         const syncData = this._syncAccumulator.getJSON();
 
         return q.all([
-            this._persistUsers(changedUsers),
+            this._persistUserPresenceEvents(userTuples),
             this._persistAccountData(syncData.accountData),
             this._persistSyncData(syncData.nextBatch, syncData.roomsData),
         ]);
@@ -239,21 +223,21 @@ LocalIndexedDBStoreBackend.prototype = {
     },
 
     /**
-     * Persist a list of User objects. Users with the same 'userId' will be
+     * Persist a list of [user id, presence event] they are for.
-     * replaced.
+     * Users with the same 'userId' will be replaced.
-     * @param {User[]} users An array of users
+     * Prfesence events should be the event in its raw form (not the Event
+     * object)
+     * @param {Object[]} users An array of users
      * @return {Promise} Resolves if the users were persisted.
      */
-    _persistUsers: function(users) {
+    _persistUserPresenceEvents: function(tuples) {
         return q.try(() => {
             const txn = this.db.transaction(["users"], "readwrite");
             const store = txn.objectStore("users");
-            for (let i = 0; i < users.length; i++) {
+            for (const tuple of tuples) {
                 store.put({
-                    userId: users[i].userId,
+                    userId: tuple[0],
-                    event: (users[i].events.presence ?
+                    event: tuple[1],
-                                users[i].events.presence.event :
-                                null),
                 }); // put == UPSERT
             }
             return promiseifyTxn(txn);
@@ -261,20 +245,17 @@ LocalIndexedDBStoreBackend.prototype = {
     },
 
     /**
-     * Load all the users from the database. This is not cached.
+     * Load all user presence events from the database. This is not cached.
-     * @return {Promise<User[]>} A list of users.
+     * FIXME: It would probably be more sensible to store the events in the
+     * sync.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
      */
-    loadUsers: function() {
+    loadUserPresenceEvents: function() {
         return q.try(() => {
             const txn = this.db.transaction(["users"], "readonly");
             const store = txn.objectStore("users");
             return selectQuery(store, undefined, (cursor) => {
-                const user = new User(cursor.value.userId);
+                return [cursor.value.userId, cursor.value.event];
-                if (cursor.value.event) {
-                    user.setPresenceEvent(new MatrixEvent(cursor.value.event));
-                }
-                this._userModifiedMap[user.userId] = user.getLastModifiedTime();
-                return user;
             });
         });
     },

src/store/indexeddb-remote-backend.js

@@ -0,0 +1,157 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+
+/**
+ * An IndexedDB store backend where the actual backend sits in a web
+ * worker.
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {string} workerScript URL to the worker script
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ */
+const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
+    workerScript, dbName,
+) {
+    this._worker = new Worker(workerScript);
+    this._nextSeq = 0;
+    // The currently in-flight requests to the actual backend
+    this._inFlight = {
+        // seq: promise,
+    };
+
+    this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+    this._doCmd('_setupWorker', [dbName]).done(() => {
+        console.log("IndexedDB worker is ready");
+    });
+};
+
+
+RemoteIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        return this._doCmd('connect');
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        return this._doCmd('clearDatabase');
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return this._doCmd('getSavedSync');
+    },
+
+    setSyncData: function(syncData) {
+        return this._doCmd('setSyncData', [syncData]);
+    },
+
+    syncToDatabase: function(users) {
+        return this._doCmd('syncToDatabase', [users]);
+    },
+
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    loadUserPresenceEvents: function() {
+        return this._doCmd('loadUserPresenceEvents');
+    },
+
+    /**
+     * Load all the account data events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of raw global account events.
+     */
+    loadAccountData: function() {
+        return this._doCmd('loadAccountData');
+    },
+
+    /**
+     * Load the sync data from the database.
+     * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
+     */
+    loadSyncData: function() {
+        return this._doCmd('loadSyncData');
+    },
+
+    _doCmd: function(cmd, args) {
+        // wrap in a q so if the postMessage throws,
+        // the promise automatically gets rejected
+        return q().then(() => {
+            const seq = this._nextSeq++;
+            const def = q.defer();
+
+            this._inFlight[seq] = def;
+
+            this._worker.postMessage({
+                command: cmd,
+                seq: seq,
+                args: args,
+            });
+
+            return def.promise;
+        }).catch((e) => {
+            console.log(args[0]);
+        });
+    },
+
+    _onWorkerMessage: function(ev) {
+        const msg = ev.data;
+
+        if (msg.command == 'cmd_success' || msg.command == 'cmd_fail') {
+            if (msg.seq === undefined) {
+                console.error("Got reply from worker with no seq");
+                return;
+            }
+
+            const def = this._inFlight[msg.seq];
+            if (def === undefined) {
+                console.error("Got reply for unknown seq " + msg.seq);
+                return;
+            }
+            delete this._inFlight[msg.seq];
+
+            if (msg.command == 'cmd_success') {
+                def.resolve(msg.result);
+            } else {
+                def.reject(msg.error);
+            }
+        } else {
+            console.warn("Unrecognised message from worker: " + msg);
+        }
+    },
+};
+
+export default RemoteIndexedDBStoreBackend;

src/store/indexeddb-remote-worker.js

@@ -0,0 +1,104 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import q from "q";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+
+/**
+ * This class lives in the webworker and drives a LocalIndexedDbStoreBackend
+ * controlled by messages from the main process.
+ */
+class IndexedDbStoreWorker {
+    constructor(postMessage) {
+        this.backend = null;
+        this.postMessage = postMessage;
+
+        this.onMessage = this.onMessage.bind(this);
+    }
+
+    onMessage(ev) {
+        const msg = ev.data;
+        let prom;
+
+        switch (msg.command) {
+            case '_setupWorker':
+                this.backend = new LocalIndexedDBStoreBackend(
+                    // this is the 'indexedDB' global (where global != window
+                    // because it's a web worker and there is no window).
+                    indexedDB, msg.args[0],
+                );
+                prom = q();
+                break;
+            case 'connect':
+                prom = this.backend.connect();
+                break;
+            case 'clearDatabase':
+                prom = this.backend.clearDatabase().then((result) => {
+                    // This returns special classes which can't be cloned
+                    // accross to the main script, so don't try.
+                    return {};
+                });
+                break;
+            case 'getSavedSync':
+                prom = this.backend.getSavedSync(false);
+                break;
+            case 'setSyncData':
+                prom = this.backend.setSyncData(...msg.args);
+                break;
+            case 'syncToDatabase':
+                prom = this.backend.syncToDatabase(...msg.args).then(() => {
+                    // This also returns IndexedDB events which are not cloneable
+                    return {};
+                });
+                break;
+            case 'loadUserPresenceEvents':
+                prom = this.backend.loadUserPresenceEvents();
+                break;
+            case 'loadAccountData':
+                prom = this.backend.loadAccountData();
+                break;
+            case 'loadSyncData':
+                prom = this.backend.loadSyncData();
+                break;
+        }
+
+        if (prom === undefined) {
+            postMessage({
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Canb't be an Error because they're not structured cloneable
+                error: "Unrecognised command",
+            });
+            return;
+        }
+
+        prom.done((ret) => {
+            this.postMessage.call(null, {
+                command: 'cmd_success',
+                seq: msg.seq,
+                result: ret,
+            });
+        }, (err) => {
+            this.postMessage.call(null, {
+                command: 'cmd_fail',
+                seq: msg.seq,
+                error: err,
+            });
+        });
+    }
+}
+
+module.exports = IndexedDbStoreWorker;

src/store/indexeddb.js

@@ -18,6 +18,9 @@ import q from "q";
 import {MatrixInMemoryStore} from "./memory";
 import utils from "../utils";
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+import RemoteIndexedDBStoreBackend from "./indexeddb-remote-backend.js";
+import User from "../models/user";
+import {MatrixEvent} from "../models/event";
 
 /**
  * This is an internal module. See {@link IndexedDBStore} for the public class.
@@ -64,6 +67,10 @@ const WRITE_DELAY_MS = 1000 * 60 * 5; // once every 5 minutes
  * <code>window.indexedDB</code>
  * @param {string=} opts.dbName Optional database name. The same name must be used
  * to open the same database.
+ * @param {string=} opts.workerScript Optional URL to a script to invooke a web
+ * worker with to run IndexedDB queries on the web worker. The IndexedDbStoreWorker
+ * class is provided for this purpose and requires the application to provide a
+ * trivial wrapper script around it.
  * @prop {IndexedDBStoreBackend} backend The backend instance. Call through to
  * this API if you need to perform specific indexeddb actions like deleting the
  * database.
@@ -75,9 +82,21 @@ const IndexedDBStore = function IndexedDBStore(opts) {
         throw new Error('Missing required option: indexedDB');
     }
 
-    this.backend = new LocalIndexedDBStoreBackend(opts.indexedDB, opts.dbName);
+    if (opts.workerScript) {
+        this.backend = new RemoteIndexedDBStoreBackend(opts.workerScript, opts.dbName);
+    } else {
+        this.backend = new LocalIndexedDBStoreBackend(opts.indexedDB, opts.dbName);
+    }
+
     this.startedUp = false;
     this._syncTs = Date.now(); // updated when writes to the database are performed
+
+    // Records the last-modified-time of each user at the last point we saved
+    // the database, such that we can derive the set if users that have been
+    // modified since we last saved.
+    this._userModifiedMap = {
+        // user_id : timestamp
+    };
 };
 utils.inherits(IndexedDBStore, MatrixInMemoryStore);
 
@@ -90,22 +109,27 @@ IndexedDBStore.prototype.startup = function() {
     }
     return this.backend.connect().then(() => {
         return q.all([
-            this.backend.loadUsers(),
+            this.backend.loadUserPresenceEvents(),
             this.backend.loadAccountData(),
             this.backend.loadSyncData(),
         ]);
     }).then((values) => {
-        const [users, accountData, syncData] = values;
+        const [userPresenceEvents, accountData, syncData] = values;
         console.log(
             "Loaded data from database: sync from ", syncData.nextBatch,
             " -- Reticulating splines...",
         );
-        users.forEach((u) => {
+        userPresenceEvents.forEach(([userId, rawEvent]) => {
+            const u = new User(userId);
+            if (rawEvent) {
+                u.setPresenceEvent(new MatrixEvent(rawEvent));
+            }
+            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
             this.storeUser(u);
         });
         this._syncTs = Date.now(); // pretend we've written so we don't rewrite
         this.setSyncToken(syncData.nextBatch);
-        this.setSyncData({
+        return this.setSyncData({
             next_batch: syncData.nextBatch,
             rooms: syncData.roomsData,
             account_data: {
@@ -146,7 +170,21 @@ IndexedDBStore.prototype.save = function() {
     const now = Date.now();
     if (now - this._syncTs > WRITE_DELAY_MS) {
         this._syncTs = Date.now(); // set now to guard against multi-writes
-        return this.backend.syncToDatabase(this.getUsers()).catch((err) => {
+
+        // work out changed users (this doesn't handle deletions but you
+        // can't 'delete' users as they are just presence events).
+        const userTuples = [];
+        for (const u of this.getUsers()) {
+            if (this._userModifiedMap[u.userId] === u.getLastModifiedTime()) continue;
+            if (!u.events.presence) continue;
+
+            userTuples.push([u.userId, u.events.presence.event]);
+
+            // note that we've saved this version of the user
+            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+        }
+
+        return this.backend.syncToDatabase(userTuples).catch((err) => {
             console.error("sync fail:", err);
         });
     }
@@ -154,7 +192,7 @@ IndexedDBStore.prototype.save = function() {
 };
 
 IndexedDBStore.prototype.setSyncData = function(syncData) {
-    this.backend.setSyncData(syncData);
+    return this.backend.setSyncData(syncData);
 };
 
 module.exports.IndexedDBStore = IndexedDBStore;

src/sync.js

@@ -573,7 +573,7 @@ SyncApi.prototype._sync = function(syncOptions) {
             );
             return this._currentSyncRequest;
         }
-    }).done(function(data) {
+    }).then(function(data) {
         //debuglog('Completed sync, next_batch=' + data.next_batch);
 
         // set the sync token NOW *before* processing the events. We do this so
@@ -584,6 +584,19 @@ SyncApi.prototype._sync = function(syncOptions) {
         // Reset after a successful sync
         self._failedSyncCount = 0;
 
+        // We need to wait until the sync data has been sent to the backend
+        // because it appears that the sync data gets modified somewhere in
+        // processing it in such a way as to make it no longer cloneable.
+        // XXX: Find out what is modifying it!
+        if (!isCachedResponse) {
+            // Don't give the store back its own cached data
+            return client.store.setSyncData(data).then(() => {
+                return data;
+            });
+        } else {
+            return q(data);
+        }
+    }).done((data) => {
         try {
             self._processSyncResponse(syncToken, data);
         } catch (e) {
@@ -592,11 +605,6 @@ SyncApi.prototype._sync = function(syncOptions) {
             console.error("Caught /sync error", e.stack || e);
         }
 
-        // Don't give the store back its own cached data
-        if (!isCachedResponse) {
-            client.store.setSyncData(data);
-        }
-
         // emit synced events
         const syncEventData = {
             oldSyncToken: syncToken,