diff --git a/lib/store/webstorage.js b/lib/store/webstorage.js index 7d2f90667..8f1d81618 100644 --- a/lib/store/webstorage.js +++ b/lib/store/webstorage.js @@ -4,7 +4,6 @@ *
* Room data is stored as follows: * room_$ROOMID_timeline_$INDEX : [ Event, Event, Event ] - * room_$ROOMID_indexes : {event_id: index} * room_$ROOMID_state : { * pagination_token:, * events: { @@ -28,20 +27,19 @@ * * Retrieval of earlier messages * ----------------------------- - * Retrieving earlier messages requires a Room which then finds the earliest - * event_id (E) in the timeline for the given Room instance. E is then mapped - * to an index I in room_$ROOMID_indexes. I is then retrieved from + * The earliest event the Room instance knows about is E. Retrieving earlier + * messages requires a Room which has a storageToken defined. + * This token maps to the index I where the Room is at. Events are then retrieved from * room_$ROOMID_timeline_{I} and events after E are extracted. If the limit * demands more events, I+1 is retrieved, up until I=max $INDEX where it gives * less than the limit. * * Full Insertion * -------------- - * Storing a room requires the timeline, indexes and state keys for $ROOMID to + * Storing a room requires the timeline and state keys for $ROOMID to * be blown away and completely replaced, which is computationally expensive. * Room.timeline is batched according to the given batch size B. These batches - * are then inserted into storage as room_$ROOMID_timeline_$INDEX. Indexes for - * the events in each batch are also persisted to room_$ROOMID_indexes. Finally, + * are then inserted into storage as room_$ROOMID_timeline_$INDEX. Finally, * the current room state is persisted to room_$ROOMID_state. * * Incremental Insertion @@ -62,7 +60,11 @@ * the earliest entries are inserted into the $INDEX (the earliest entries are * inverted in _live, so the earliest entry is at index 0, not len-1) until the * batch == B. Then, the remaining entries in _live are batched to $INDEX-1, - * $INDEX-2, and so on. This will result in negative indices. + * $INDEX-2, and so on. This will result in negative indices. The easiest way to + * visualise this is that the timeline goes from new to old, left to right: + * -2 -1 0 1 + * <--NEW---------------------------------------OLD--> + * [a,b,c] [d,e,f] [g,h,i] [j,k,l] * * Purging * ------- @@ -77,9 +79,6 @@ * room_!foo:bar_timeline_0 : [M1, M2, M3, M4] * room_!foo:bar_timeline_1 : [M5, M6, M7, M8] * room_!foo:bar_timeline_2 : [M9] - * room_!foo:bar_indexes : { M1: 0, M2: 0, M3: 0, M4: 0, - * M5: 1, M6: 1, M7: 1, M8: 1, - * M9: 2 } * room_!foo:bar_state: { ... } * * 5 new messages (N1-5, 1=newest) arrive and are then added: [Incremental Insert] @@ -93,14 +92,14 @@ * room_!foo:bar_timeline_-1 : [N2, N3, N4, N5] * room_!foo:bar_timeline_-2 : [N1] * room_!foo:bar_timeline_live: [] - * room_!foo:bar_indexes : {N1: -2, N2: -1, ...} * * And the room is retrieved with 8 messages: [Room Retrieval] * Room.timeline: [N1, N2, N3, N4, N5, M1, M2, M3] + * Room.storageToken: => early_index 0 * * 3 earlier messages are requested: [Earlier retrieval] + * Use storageToken to find batch index 0. Scan batch for earliest event ID. * earliest event = M3 - * index = room_!foo:bar_indexes[M3] = 0 * events = room_!foo:bar_timeline[0] where event > M3 = [M4] * Too few events, use next index and get 2 more: * events = room_!foo:bar_timeline[1] = [M5, M6, M7, M8] => [M5, M6] @@ -112,6 +111,8 @@ */ var utils = require("../utils"); +var Room = require("../models/room"); +var MatrixEvent = require("../models/event").MatrixEvent; /** * Construct a web storage store, capable of storing rooms and users. @@ -161,16 +162,25 @@ WebStorageStore.prototype.setSyncToken = function(token) { * @param {Room} room */ WebStorageStore.prototype.storeRoom = function(room) { - initRoomStruct(this.store, room); + var serRoom = SerialisedRoom.fromRoom(room, this.batchSize); + persist(this.store, serRoom); }; /** * Retrieve a room from web storage. * @param {string} roomId - * @return {null} + * @return {?Room} */ WebStorageStore.prototype.getRoom = function(roomId) { - return null; + // probe if room exists; break early if not. Every room should have state. + if (!this.store.getItem(keyName(roomId, "state"))) { + return null; + } + var timelineKeys = getTimelineIndices(this.store, roomId); + if (timelineKeys.indexOf("live") !== -1) { + this._syncTimeline(roomId, timelineKeys); + } + return loadRoom(this.store, roomId, this.batchSize); }; /** @@ -216,11 +226,180 @@ WebStorageStore.prototype.scrollback = function(room, limit) { return []; }; -function initRoomStruct(store, roomId) { - var prefix = "room_" + roomId; - store.setItem(prefix + "_timeline_0", []); - store.setItem(prefix + "_indexes", {}); - store.setItem(prefix + "_state", {}); +/** + * Sync the 'live' timeline, batching live events according to 'batchSize'. + * @param {string} roomId The room to sync the timeline. + * @param {Array } timelineIndices Optional. The indices in the timeline + * if known already. + */ +WebStorageStore.prototype._syncTimeline = function(roomId, timelineIndices) { + timelineIndices = timelineIndices || getTimelineIndices(this.store, roomId); + var liveEvents = this.store.getItem(keyName(roomId, "timeline", "live")) || []; + + // get the lowest numbered $INDEX batch + var lowestIndex = getLowestIndex(timelineIndices); + var lowKey = keyName(roomId, "timeline", lowestIndex); + var lowestBatch = this.store.getItem(lowKey) || []; + + // fill up the existing batch first. + while (lowestBatch.length < this.batchSize && liveEvents.length > 0) { + lowestBatch.unshift(liveEvents.shift()); + } + this.store.setItem(lowKey, lowestBatch); + + // start adding new batches as required + var batch = []; + while (liveEvents.length > 0) { + batch.unshift(liveEvents.shift()); + if (batch.length === this.batchSize) { + // persist the full batch and make another + lowestIndex--; + lowKey = keyName(roomId, "timeline", lowestIndex); + this.store.setItem(lowKey, batch); + batch = []; + } + } + // reset live array + this.store.setItem(keyName(roomId, "timeline", "live"), []); +}; + +function SerialisedRoom(roomId) { + this.state = { + events: {} + }; + this.timeline = { + // $INDEX: [] + }; + this.roomId = roomId; +} + +/** + * Convert a Room instance into a SerialisedRoom instance which can be stored + * in the key value store. + * @param {Room} room The matrix room to convert + * @param {integer} batchSize The number of events per timeline batch + * @return {SerialisedRoom} A serialised room representation of 'room'. + */ +SerialisedRoom.fromRoom = function(room, batchSize) { + var self = new SerialisedRoom(room.roomId); + var i, ptr; + self.state.pagination_token = room.oldState.paginationToken; + // [room_$ROOMID_state] downcast to POJO from MatrixEvent + utils.forEach(utils.keys(room.currentState.events), function(eventType) { + utils.forEach(utils.keys(room.currentState.events[eventType]), function(skey) { + if (!self.state.events[eventType]) { + self.state.events[eventType] = {}; + } + self.state.events[eventType][skey] = ( + room.currentState.events[eventType][skey].event + ); + }); + }); + + // [room_$ROOMID_timeline_$INDEX] + if (batchSize > 0) { + ptr = 0; + while (ptr * batchSize < room.timeline.length) { + self.timeline[ptr] = room.timeline.slice( + ptr * batchSize, (ptr + 1) * batchSize + ); + self.timeline[ptr] = utils.map(self.timeline[ptr][i], function(me) { + // use POJO not MatrixEvent + return me.event; + }); + ptr++; + } + } + else { // don't batch + self.timeline[0] = utils.map(room.timeline, function(matrixEvent) { + return matrixEvent.event; + }); + } + return self; +}; + +function loadRoom(store, roomId, numEvents) { + var room = new Room(roomId); + // populate state (flatten nested struct to event array) + var currentStateMap = store.getItem(keyName(roomId, "state")); + var stateEvents = []; + utils.forEach(utils.keys(currentStateMap.events), function(eventType) { + utils.forEach(utils.keys(currentStateMap.events[eventType]), function(skey) { + stateEvents.push(currentStateMap[eventType][skey]); + }); + }); + // TODO: Fix logic dupe with MatrixClient._processRoomEvents + var oldStateEvents = utils.map( + utils.deepCopy(stateEvents), function(e) { + return new MatrixEvent(e); + } + ); + var currentStateEvents = utils.map(stateEvents, function(e) { + return new MatrixEvent(e); + } + ); + room.oldState.setStateEvents(oldStateEvents); + room.currentState.setStateEvents(currentStateEvents); + + // add most recent numEvents + var recentEvents = []; + var index = getLowestIndex(getTimelineIndices(store, roomId)); + var i, key, batch; + while (recentEvents.length < numEvents) { + key = keyName(roomId, "timeline", index); + batch = store.getItem(key) || []; + if (batch.length === 0) { + // nothing left in the store. + break; + } + for (i = 0; i < batch.length; i++) { + recentEvents.unshift(new MatrixEvent(batch[i])); + } + } + room.addEventsToTimeline(recentEvents.reverse(), true); + room.oldState.paginationToken = currentStateMap.pagination_token; + return room; +} + +function persist(store, serRoom) { + store.setItem(keyName(serRoom.roomId, "state"), serRoom.state); + utils.keys(serRoom.timeline, function(index) { + store.setItem( + keyName(serRoom.roomId, "timeline", index), + serRoom.timeline[index] + ); + }); +} + +function getTimelineIndices(store, roomId) { + var keys = []; + for (var i = 0; i < store.length; i++) { + if (store.key(i).indexOf(keyName(roomId, "timeline_")) !== -1) { + // e.g. room_$ROOMID_timeline_0 => 0 + keys.push( + store.key(i).replace(keyName(roomId, "timeline_"), "") + ); + } + } + return keys; +} + +function getLowestIndex(timelineIndices) { + var lowestIndex = 0; + var index; + for (var i = 0; i < timelineIndices.length; i++) { + index = parseInt(timelineIndices[i]); + if (index && index < lowestIndex) { + lowestIndex = index; + } + } + return lowestIndex; +} + +function keyName(roomId, key, index) { + return "room_" + roomId + "_" + key + ( + index === undefined ? "" : ("_" + index) + ); } /* diff --git a/spec/unit/webstorage.spec.js b/spec/unit/webstorage.spec.js new file mode 100644 index 000000000..d286e6365 --- /dev/null +++ b/spec/unit/webstorage.spec.js @@ -0,0 +1,77 @@ +"use strict"; +var sdk = require("../.."); +var WebStorageStore = sdk.WebStorageStore; +var Room = sdk.Room; +var utils = require("../test-utils"); + +function MockStorageApi() { + this.data = {}; + this.keys = []; + this.length = 0; +} +MockStorageApi.prototype = { + setItem: function(k, v) { + this.data[k] = v; + this._recalc(); + }, + getItem: function(k) { + return this.data[k] || null; + }, + removeItem: function(k) { + delete this.data[k]; + this._recalc(); + }, + key: function(index) { + return this.keys[index]; + }, + _recalc: function() { + var keys = []; + for (var k in this.data) { + if (!this.data.hasOwnProperty(k)) { continue; } + keys.push(k); + } + this.keys = keys; + this.length = keys.length; + } +}; + +describe("WebStorageStore", function() { + var store, room; + var roomId = "!foo:bar"; + var userId = "@alice:bar"; + var mockStorageApi; + var batchNum = 3; + + beforeEach(function() { + utils.beforeEach(this); + mockStorageApi = new MockStorageApi(); + store = new WebStorageStore(mockStorageApi, batchNum); + room = new Room(roomId); + }); + + describe("storeRoom", function() { + it("should persist the room state correctly", function() { + var stateEvents = [ + utils.mkEvent({ + event: true, type: "m.room.create", user: userId, room: roomId, + content: { + creator: userId + } + }), + utils.mkMembership({ + event: true, user: userId, room: roomId, mship: "join" + }) + ]; + room.currentState.setStateEvents(stateEvents); + store.storeRoom(room); + var storedEvents = mockStorageApi.getItem( + "room_" + roomId + "_state" + ).events; + expect(storedEvents["m.room.create"][""]).toEqual(stateEvents[0].event); + }); + + xit("should persist timeline events correctly", function() { + + }); + }); +});