Add first cut webstorage implementation. Add very basic test.

lib/store/webstorage.js

@@ -4,7 +4,6 @@
  * <pre>
  * Room data is stored as follows:
  *   room_$ROOMID_timeline_$INDEX : [ Event, Event, Event ]
- *   room_$ROOMID_indexes : {event_id: index}
  *   room_$ROOMID_state : {
  *                          pagination_token: <oldState.paginationToken>,
  *                          events: {
@@ -28,20 +27,19 @@
  *
  * Retrieval of earlier messages
  * -----------------------------
- * Retrieving earlier messages requires a Room which then finds the earliest
+ * The earliest event the Room instance knows about is E. Retrieving earlier
- * event_id (E) in the timeline for the given Room instance. E is then mapped
+ * messages requires a Room which has a storageToken defined.
- * to an index I in room_$ROOMID_indexes. I is then retrieved from
+ * This token maps to the index I where the Room is at. Events are then retrieved from
  * room_$ROOMID_timeline_{I} and events after E are extracted. If the limit
  * demands more events, I+1 is retrieved, up until I=max $INDEX where it gives
  * less than the limit.
  *
  * Full Insertion
  * --------------
- * Storing a room requires the timeline, indexes and state keys for $ROOMID to
+ * Storing a room requires the timeline and state keys for $ROOMID to
  * be blown away and completely replaced, which is computationally expensive.
  * Room.timeline is batched according to the given batch size B. These batches
- * are then inserted into storage as room_$ROOMID_timeline_$INDEX. Indexes for
+ * are then inserted into storage as room_$ROOMID_timeline_$INDEX. Finally,
- * the events in each batch are also persisted to room_$ROOMID_indexes. Finally,
  * the current room state is persisted to room_$ROOMID_state.
  *
  * Incremental Insertion
@@ -62,7 +60,11 @@
  * the earliest entries are inserted into the $INDEX (the earliest entries are
  * inverted in _live, so the earliest entry is at index 0, not len-1) until the
  * batch == B. Then, the remaining entries in _live are batched to $INDEX-1,
- * $INDEX-2, and so on. This will result in negative indices.
+ * $INDEX-2, and so on. This will result in negative indices. The easiest way to
+ * visualise this is that the timeline goes from new to old, left to right:
+ *          -2         -1         0         1
+ * <--NEW---------------------------------------OLD-->
+ *        [a,b,c]    [d,e,f]   [g,h,i]   [j,k,l]
  *
  * Purging
  * -------
@@ -77,9 +79,6 @@
  *   room_!foo:bar_timeline_0 : [M1, M2, M3, M4]
  *   room_!foo:bar_timeline_1 : [M5, M6, M7, M8]
  *   room_!foo:bar_timeline_2 : [M9]
- *   room_!foo:bar_indexes : { M1: 0, M2: 0, M3: 0, M4: 0,
- *                             M5: 1, M6: 1, M7: 1, M8: 1,
- *                             M9: 2 }
  *   room_!foo:bar_state: { ... }
  *
  * 5 new messages (N1-5, 1=newest) arrive and are then added: [Incremental Insert]
@@ -93,14 +92,14 @@
  *   room_!foo:bar_timeline_-1 : [N2, N3, N4, N5]
  *   room_!foo:bar_timeline_-2 : [N1]
  *   room_!foo:bar_timeline_live: []
- *   room_!foo:bar_indexes : {N1: -2, N2: -1, ...}
  *
  * And the room is retrieved with 8 messages: [Room Retrieval]
  *   Room.timeline: [N1, N2, N3, N4, N5, M1, M2, M3]
+ *   Room.storageToken: => early_index 0
  *
  * 3 earlier messages are requested: [Earlier retrieval]
+ *   Use storageToken to find batch index 0. Scan batch for earliest event ID.
  *   earliest event = M3
- *   index = room_!foo:bar_indexes[M3] = 0
  *   events = room_!foo:bar_timeline[0] where event > M3 = [M4]
  * Too few events, use next index and get 2 more:
  *   events = room_!foo:bar_timeline[1] = [M5, M6, M7, M8] => [M5, M6]
@@ -112,6 +111,8 @@
  */
 
 var utils = require("../utils");
+var Room = require("../models/room");
+var MatrixEvent = require("../models/event").MatrixEvent;
 
 /**
  * Construct a web storage store, capable of storing rooms and users.
@@ -161,16 +162,25 @@ WebStorageStore.prototype.setSyncToken = function(token) {
  * @param {Room} room
  */
 WebStorageStore.prototype.storeRoom = function(room) {
-    initRoomStruct(this.store, room);
+    var serRoom = SerialisedRoom.fromRoom(room, this.batchSize);
+    persist(this.store, serRoom);
 };
 
 /**
  * Retrieve a room from web storage.
  * @param {string} roomId
- * @return {null}
+ * @return {?Room}
  */
 WebStorageStore.prototype.getRoom = function(roomId) {
-    return null;
+    // probe if room exists; break early if not. Every room should have state.
+    if (!this.store.getItem(keyName(roomId, "state"))) {
+        return null;
+    }
+    var timelineKeys = getTimelineIndices(this.store, roomId);
+    if (timelineKeys.indexOf("live") !== -1) {
+        this._syncTimeline(roomId, timelineKeys);
+    }
+    return loadRoom(this.store, roomId, this.batchSize);
 };
 
 /**
@@ -216,11 +226,180 @@ WebStorageStore.prototype.scrollback = function(room, limit) {
     return [];
 };
 
-function initRoomStruct(store, roomId) {
+/**
-    var prefix = "room_" + roomId;
+ * Sync the 'live' timeline, batching live events according to 'batchSize'.
-    store.setItem(prefix + "_timeline_0", []);
+ * @param {string} roomId The room to sync the timeline.
-    store.setItem(prefix + "_indexes", {});
+ * @param {Array<String>} timelineIndices Optional. The indices in the timeline
-    store.setItem(prefix + "_state", {});
+ * if known already.
+ */
+WebStorageStore.prototype._syncTimeline = function(roomId, timelineIndices) {
+    timelineIndices = timelineIndices || getTimelineIndices(this.store, roomId);
+    var liveEvents = this.store.getItem(keyName(roomId, "timeline", "live")) || [];
+
+    // get the lowest numbered $INDEX batch
+    var lowestIndex = getLowestIndex(timelineIndices);
+    var lowKey = keyName(roomId, "timeline", lowestIndex);
+    var lowestBatch = this.store.getItem(lowKey) || [];
+
+    // fill up the existing batch first.
+    while (lowestBatch.length < this.batchSize && liveEvents.length > 0) {
+        lowestBatch.unshift(liveEvents.shift());
+    }
+    this.store.setItem(lowKey, lowestBatch);
+
+    // start adding new batches as required
+    var batch = [];
+    while (liveEvents.length > 0) {
+        batch.unshift(liveEvents.shift());
+        if (batch.length === this.batchSize) {
+            // persist the full batch and make another
+            lowestIndex--;
+            lowKey = keyName(roomId, "timeline", lowestIndex);
+            this.store.setItem(lowKey, batch);
+            batch = [];
+        }
+    }
+    // reset live array
+    this.store.setItem(keyName(roomId, "timeline", "live"), []);
+};
+
+function SerialisedRoom(roomId) {
+    this.state = {
+        events: {}
+    };
+    this.timeline = {
+        // $INDEX: []
+    };
+    this.roomId = roomId;
+}
+
+/**
+ * Convert a Room instance into a SerialisedRoom instance which can be stored
+ * in the key value store.
+ * @param {Room} room The matrix room to convert
+ * @param {integer} batchSize The number of events per timeline batch
+ * @return {SerialisedRoom} A serialised room representation of 'room'.
+ */
+SerialisedRoom.fromRoom = function(room, batchSize) {
+    var self = new SerialisedRoom(room.roomId);
+    var i, ptr;
+    self.state.pagination_token = room.oldState.paginationToken;
+    // [room_$ROOMID_state] downcast to POJO from MatrixEvent
+    utils.forEach(utils.keys(room.currentState.events), function(eventType) {
+        utils.forEach(utils.keys(room.currentState.events[eventType]), function(skey) {
+            if (!self.state.events[eventType]) {
+                self.state.events[eventType] = {};
+            }
+            self.state.events[eventType][skey] = (
+                room.currentState.events[eventType][skey].event
+            );
+        });
+    });
+
+    // [room_$ROOMID_timeline_$INDEX]
+    if (batchSize > 0) {
+        ptr = 0;
+        while (ptr * batchSize < room.timeline.length) {
+            self.timeline[ptr] = room.timeline.slice(
+                ptr * batchSize, (ptr + 1) * batchSize
+            );
+            self.timeline[ptr] = utils.map(self.timeline[ptr][i], function(me) {
+                // use POJO not MatrixEvent
+                return me.event;
+            });
+            ptr++;
+        }
+    }
+    else { // don't batch
+        self.timeline[0] = utils.map(room.timeline, function(matrixEvent) {
+            return matrixEvent.event;
+        });
+    }
+    return self;
+};
+
+function loadRoom(store, roomId, numEvents) {
+    var room = new Room(roomId);
+    // populate state (flatten nested struct to event array)
+    var currentStateMap = store.getItem(keyName(roomId, "state"));
+    var stateEvents = [];
+    utils.forEach(utils.keys(currentStateMap.events), function(eventType) {
+        utils.forEach(utils.keys(currentStateMap.events[eventType]), function(skey) {
+            stateEvents.push(currentStateMap[eventType][skey]);
+        });
+    });
+    // TODO: Fix logic dupe with MatrixClient._processRoomEvents
+    var oldStateEvents = utils.map(
+        utils.deepCopy(stateEvents), function(e) {
+            return new MatrixEvent(e);
+        }
+    );
+    var currentStateEvents = utils.map(stateEvents, function(e) {
+            return new MatrixEvent(e);
+        }
+    );
+    room.oldState.setStateEvents(oldStateEvents);
+    room.currentState.setStateEvents(currentStateEvents);
+
+    // add most recent numEvents
+    var recentEvents = [];
+    var index = getLowestIndex(getTimelineIndices(store, roomId));
+    var i, key, batch;
+    while (recentEvents.length < numEvents) {
+        key = keyName(roomId, "timeline", index);
+        batch = store.getItem(key) || [];
+        if (batch.length === 0) {
+            // nothing left in the store.
+            break;
+        }
+        for (i = 0; i < batch.length; i++) {
+            recentEvents.unshift(new MatrixEvent(batch[i]));
+        }
+    }
+    room.addEventsToTimeline(recentEvents.reverse(), true);
+    room.oldState.paginationToken = currentStateMap.pagination_token;
+    return room;
+}
+
+function persist(store, serRoom) {
+    store.setItem(keyName(serRoom.roomId, "state"), serRoom.state);
+    utils.keys(serRoom.timeline, function(index) {
+        store.setItem(
+            keyName(serRoom.roomId, "timeline", index),
+            serRoom.timeline[index]
+        );
+    });
+}
+
+function getTimelineIndices(store, roomId) {
+    var keys = [];
+    for (var i = 0; i < store.length; i++) {
+        if (store.key(i).indexOf(keyName(roomId, "timeline_")) !== -1) {
+            // e.g. room_$ROOMID_timeline_0  =>  0
+            keys.push(
+                store.key(i).replace(keyName(roomId, "timeline_"), "")
+            );
+        }
+    }
+    return keys;
+}
+
+function getLowestIndex(timelineIndices) {
+    var lowestIndex = 0;
+    var index;
+    for (var i = 0; i < timelineIndices.length; i++) {
+        index = parseInt(timelineIndices[i]);
+        if (index && index < lowestIndex) {
+            lowestIndex = index;
+        }
+    }
+    return lowestIndex;
+}
+
+function keyName(roomId, key, index) {
+    return "room_" + roomId + "_" + key + (
+        index === undefined ? "" : ("_" + index)
+    );
 }
 
 /*

spec/unit/webstorage.spec.js

@@ -0,0 +1,77 @@
+"use strict";
+var sdk = require("../..");
+var WebStorageStore = sdk.WebStorageStore;
+var Room = sdk.Room;
+var utils = require("../test-utils");
+
+function MockStorageApi() {
+    this.data = {};
+    this.keys = [];
+    this.length = 0;
+}
+MockStorageApi.prototype = {
+    setItem: function(k, v) {
+        this.data[k] = v;
+        this._recalc();
+    },
+    getItem: function(k) {
+        return this.data[k] || null;
+    },
+    removeItem: function(k) {
+        delete this.data[k];
+        this._recalc();
+    },
+    key: function(index) {
+        return this.keys[index];
+    },
+    _recalc: function() {
+        var keys = [];
+        for (var k in this.data) {
+            if (!this.data.hasOwnProperty(k)) { continue; }
+            keys.push(k);
+        }
+        this.keys = keys;
+        this.length = keys.length;
+    }
+};
+
+describe("WebStorageStore", function() {
+    var store, room;
+    var roomId = "!foo:bar";
+    var userId = "@alice:bar";
+    var mockStorageApi;
+    var batchNum = 3;
+
+    beforeEach(function() {
+        utils.beforeEach(this);
+        mockStorageApi = new MockStorageApi();
+        store = new WebStorageStore(mockStorageApi, batchNum);
+        room = new Room(roomId);
+    });
+
+    describe("storeRoom", function() {
+        it("should persist the room state correctly", function() {
+            var stateEvents = [
+                utils.mkEvent({
+                    event: true, type: "m.room.create", user: userId, room: roomId,
+                    content: {
+                        creator: userId
+                    }
+                }),
+                utils.mkMembership({
+                    event: true, user: userId, room: roomId, mship: "join"
+                })
+            ];
+            room.currentState.setStateEvents(stateEvents);
+            store.storeRoom(room);
+            var storedEvents = mockStorageApi.getItem(
+                "room_" + roomId + "_state"
+            ).events;
+            expect(storedEvents["m.room.create"][""]).toEqual(stateEvents[0].event);
+        });
+
+        xit("should persist timeline events correctly", function() {
+            
+        });
+    });
+});