diff --git a/lib/store/webstorage.js b/lib/store/webstorage.js index a7fbedbe7..da3936f06 100644 --- a/lib/store/webstorage.js +++ b/lib/store/webstorage.js @@ -19,7 +19,7 @@ * -------------- * Retrieving a room requires the $ROOMID which then pulls out the current state * from room_$ROOMID_state. A defined starting batch of timeline events are then - * extracted from the lowest numbered $INDEX for room_$ROOMID_timeline_$INDEX + * extracted from the highest numbered $INDEX for room_$ROOMID_timeline_$INDEX * (more indices as required). The $INDEX may be negative. These are * added to the timeline in the same way as /initialSync (old state will diverge). * If there exists a room_$ROOMID_timeline_live key, then a timeline sync should @@ -30,9 +30,9 @@ * The earliest event the Room instance knows about is E. Retrieving earlier * messages requires a Room which has a storageToken defined. * This token maps to the index I where the Room is at. Events are then retrieved from - * room_$ROOMID_timeline_{I} and events after E are extracted. If the limit - * demands more events, I+1 is retrieved, up until I=max $INDEX where it gives - * less than the limit. + * room_$ROOMID_timeline_{I} and elements before E are extracted. If the limit + * demands more events, I-1 is retrieved, up until I=min $INDEX where it gives + * less than the limit. Index may go negative if you have paginated in the past. * * Full Insertion * -------------- @@ -45,8 +45,7 @@ * Incremental Insertion * --------------------- * As events arrive, the store can quickly persist these new events. This - * involves pushing the events to room_$ROOMID_timeline_live. This results in an - * inverted ordering where the highest number is the most recent entry. If the + * involves pushing the events to room_$ROOMID_timeline_live. If the * current room state has been modified by the new event, then * room_$ROOMID_state should be updated in addition to the timeline. * @@ -56,24 +55,23 @@ * events. This is computationally expensive to perform on every new event, so * is deferred by inserting live events to room_$ROOMID_timeline_live. A * timeline sync reconciles timeline_live and timeline_$INDEX. This involves - * retrieving _live and the lowest numbered $INDEX batch. If the batch is < B, - * the earliest entries are inserted into the $INDEX (the earliest entries are - * inverted in _live, so the earliest entry is at index 0, not len-1) until the - * batch == B. Then, the remaining entries in _live are batched to $INDEX-1, - * $INDEX-2, and so on. This will result in negative indices. The easiest way to - * visualise this is that the timeline goes from new to old, left to right: + * retrieving _live and the highest numbered $INDEX batch. If the batch is < B, + * the earliest entries from _live are inserted into the $INDEX until the + * batch == B. Then, the remaining entries in _live are batched to $INDEX+1, + * $INDEX+2, and so on. The easiest way to visualise this is that the timeline + * goes from old to new, left to right: * -2 -1 0 1 - * <--NEW---------------------------------------OLD--> + * <--OLD---------------------------------------NEW--> * [a,b,c] [d,e,f] [g,h,i] [j,k,l] * * Purging * ------- - * Events from the timeline can be purged by removing the highest + * Events from the timeline can be purged by removing the lowest * timeline_$INDEX in the store. * * Example * ------- - * A room with room_id !foo:bar has 9 messages (M1->9 where 1=newest) with a + * A room with room_id !foo:bar has 9 messages (M1->9 where 9=newest) with a * batch size of 4. The very first time, there is no entry for !foo:bar until * storeRoom() is called, which results in the keys: [Full Insert] * room_!foo:bar_timeline_0 : [M1, M2, M3, M4] @@ -81,31 +79,33 @@ * room_!foo:bar_timeline_2 : [M9] * room_!foo:bar_state: { ... } * - * 5 new messages (N1-5, 1=newest) arrive and are then added: [Incremental Insert] - * room_!foo:bar_timeline_live: [N5] - * room_!foo:bar_timeline_live: [N5, N4] - * room_!foo:bar_timeline_live: [N5, N4, N3] - * room_!foo:bar_timeline_live: [N5, N4, N3, N2] - * room_!foo:bar_timeline_live: [N5, N4, N3, N2, N1] + * 5 new messages (N1-5, 5=newest) arrive and are then added: [Incremental Insert] + * room_!foo:bar_timeline_live: [N1] + * room_!foo:bar_timeline_live: [N1, N2] + * room_!foo:bar_timeline_live: [N1, N2, N3] + * room_!foo:bar_timeline_live: [N1, N2, N3, N4] + * room_!foo:bar_timeline_live: [N1, N2, N3, N4, N5] * * App is shutdown. Restarts. The timeline is synced [Timeline Sync] - * room_!foo:bar_timeline_-1 : [N2, N3, N4, N5] - * room_!foo:bar_timeline_-2 : [N1] + * room_!foo:bar_timeline_2 : [M9, N1, N2, N3] + * room_!foo:bar_timeline_3 : [N4, N5] * room_!foo:bar_timeline_live: [] * * And the room is retrieved with 8 messages: [Room Retrieval] - * Room.timeline: [N1, N2, N3, N4, N5, M1, M2, M3] - * Room.storageToken: => early_index 0 + * Room.timeline: [M7, M8, M9, N1, N2, N3, N4, N5] + * Room.storageToken: => early_index = 1 because that's where M7 is. * * 3 earlier messages are requested: [Earlier retrieval] - * Use storageToken to find batch index 0. Scan batch for earliest event ID. - * earliest event = M3 - * events = room_!foo:bar_timeline[0] where event > M3 = [M4] - * Too few events, use next index and get 2 more: - * events = room_!foo:bar_timeline[1] = [M5, M6, M7, M8] => [M5, M6] + * Use storageToken to find batch index 1. Scan batch for earliest event ID. + * earliest event = M7 + * events = room_!foo:bar_timeline_1 where event < M7 = [M5, M6] + * Too few events, use next index (0) and get 1 more: + * events = room_!foo:bar_timeline_0 = [M1, M2, M3, M4] => [M4] + * Return concatentation: + * [M4, M5, M6] * * Purge oldest events: [Purge] - * del room_!foo:bar_timeline_2 + * del room_!foo:bar_timeline_0 * * @module store/webstorage */ @@ -178,7 +178,6 @@ WebStorageStore.prototype.getRoom = function(roomId) { } var timelineKeys = getTimelineIndices(this.store, roomId); if (timelineKeys.indexOf("live") !== -1) { - console.log("Syncing live"); this._syncTimeline(roomId, timelineKeys); } return loadRoom(this.store, roomId, this.batchSize); @@ -237,27 +236,25 @@ WebStorageStore.prototype._syncTimeline = function(roomId, timelineIndices) { timelineIndices = timelineIndices || getTimelineIndices(this.store, roomId); var liveEvents = this.store.getItem(keyName(roomId, "timeline", "live")) || []; - // get the lowest numbered $INDEX batch - var lowestIndex = getLowestIndex(timelineIndices); - var lowKey = keyName(roomId, "timeline", lowestIndex); - var lowestBatch = this.store.getItem(lowKey) || []; - console.log("Live Events = %s, Low batch = %s (i=%s;b=%s)", liveEvents.length, - lowestBatch.length, lowestIndex, this.batchSize); + // get the highest numbered $INDEX batch + var highestIndex = getHighestIndex(timelineIndices); + var hiKey = keyName(roomId, "timeline", highestIndex); + var hiBatch = this.store.getItem(hiKey) || []; // fill up the existing batch first. - while (lowestBatch.length < this.batchSize && liveEvents.length > 0) { - lowestBatch.unshift(liveEvents.shift()); + while (hiBatch.length < this.batchSize && liveEvents.length > 0) { + hiBatch.push(liveEvents.shift()); } - this.store.setItem(lowKey, lowestBatch); + this.store.setItem(hiKey, hiBatch); // start adding new batches as required var batch = []; while (liveEvents.length > 0) { - batch.unshift(liveEvents.shift()); + batch.push(liveEvents.shift()); if (batch.length === this.batchSize || liveEvents.length === 0) { // persist the full batch and make another - lowestIndex--; - lowKey = keyName(roomId, "timeline", lowestIndex); - this.store.setItem(lowKey, batch); + highestIndex++; + hiKey = keyName(roomId, "timeline", highestIndex); + this.store.setItem(hiKey, batch); batch = []; } } @@ -284,7 +281,7 @@ function SerialisedRoom(roomId) { */ SerialisedRoom.fromRoom = function(room, batchSize) { var self = new SerialisedRoom(room.roomId); - var ptr; + var index; self.state.pagination_token = room.oldState.paginationToken; // [room_$ROOMID_state] downcast to POJO from MatrixEvent utils.forEach(utils.keys(room.currentState.events), function(eventType) { @@ -300,16 +297,16 @@ SerialisedRoom.fromRoom = function(room, batchSize) { // [room_$ROOMID_timeline_$INDEX] if (batchSize > 0) { - ptr = 0; - while (ptr * batchSize < room.timeline.length) { - self.timeline[ptr] = room.timeline.slice( - ptr * batchSize, (ptr + 1) * batchSize + index = 0; + while (index * batchSize < room.timeline.length) { + self.timeline[index] = room.timeline.slice( + index * batchSize, (index + 1) * batchSize ); - self.timeline[ptr] = utils.map(self.timeline[ptr], function(me) { + self.timeline[index] = utils.map(self.timeline[index], function(me) { // use POJO not MatrixEvent return me.event; }); - ptr++; + index++; } } else { // don't batch @@ -345,7 +342,7 @@ function loadRoom(store, roomId, numEvents) { // add most recent numEvents var recentEvents = []; - var index = getLowestIndex(getTimelineIndices(store, roomId)); + var index = getHighestIndex(getTimelineIndices(store, roomId)); var i, key, batch; while (recentEvents.length < numEvents) { key = keyName(roomId, "timeline", index); @@ -354,14 +351,15 @@ function loadRoom(store, roomId, numEvents) { // nothing left in the store. break; } - for (i = 0; i < batch.length; i++) { + for (i = batch.length - 1; i >= 0; i--) { recentEvents.unshift(new MatrixEvent(batch[i])); if (recentEvents.length === numEvents) { break; } } - index++; + index--; } + // add events backwards to diverge old state correctly. room.addEventsToTimeline(recentEvents.reverse(), true); room.oldState.paginationToken = currentStateMap.pagination_token; return room; @@ -390,16 +388,15 @@ function getTimelineIndices(store, roomId) { return keys; } -function getLowestIndex(timelineIndices) { - var lowestIndex = 0; - var index; +function getHighestIndex(timelineIndices) { + var highestIndex, index; for (var i = 0; i < timelineIndices.length; i++) { index = parseInt(timelineIndices[i]); - if (index && index < lowestIndex) { - lowestIndex = index; + if (!isNaN(index) && (highestIndex === undefined || index > highestIndex)) { + highestIndex = index; } } - return lowestIndex; + return highestIndex; } function keyName(roomId, key, index) { diff --git a/spec/unit/webstorage.spec.js b/spec/unit/webstorage.spec.js index 267c118b1..f503d9e48 100644 --- a/spec/unit/webstorage.spec.js +++ b/spec/unit/webstorage.spec.js @@ -12,7 +12,6 @@ function MockStorageApi() { MockStorageApi.prototype = { setItem: function(k, v) { this.data[k] = v; - console.log("SetItem: %s => %s", k, JSON.stringify(v, undefined, 2)); this._recalc(); }, getItem: function(k) { @@ -50,6 +49,18 @@ describe("WebStorageStore", function() { room = new Room(roomId); }); + describe("constructor", function() { + it("should throw if the WebStorage API functions are missing", function() { + expect(function() { + store = new WebStorageStore({}, 5); + }).toThrow(); + expect(function() { + mockStorageApi.length = undefined; + store = new WebStorageStore(mockStorageApi, 5); + }).toThrow(); + }); + }); + describe("syncToken", function() { it("get: should return the token from the store", function() { var token = "flibble"; @@ -110,6 +121,31 @@ describe("WebStorageStore", function() { } } }); + + it("should persist timeline events in one bucket if batchNum=0", function() { + store = new WebStorageStore(mockStorageApi, 0); + var prefix = "room_" + roomId + "_timeline_"; + var timelineEvents = []; + var entries = batchNum + batchNum - 1; + var i = 0; + for (i = 0; i < entries; i++) { + timelineEvents.push( + utils.mkMessage({room: roomId, user: userId, event: true}) + ); + } + room.timeline = timelineEvents; + store.storeRoom(room); + expect(mockStorageApi.getItem(prefix + "-1")).toBe(null); + expect(mockStorageApi.getItem(prefix + "1")).toBe(null); + expect(mockStorageApi.getItem(prefix + "live")).toBe(null); + var timeline = mockStorageApi.getItem(prefix + "0"); + expect(timeline.length).toEqual(timelineEvents.length); + for (i = 0; i < timeline.length; i++) { + expect(timeline[i]).toEqual( + timelineEvents[i].event + ); + } + }); }); describe("getRoom", function() { @@ -133,14 +169,18 @@ describe("WebStorageStore", function() { ); // stored timeline events - var timeline0 = []; - var timeline1 = []; - for (var i = 0; i < batchNum; i++) { - timeline0[i] = utils.mkMessage({user: userId, room: roomId}); - if (i !== (batchNum - 1)) { // miss last one + var timeline0, timeline1, i; + + beforeEach(function() { + timeline0 = []; + timeline1 = []; + for (i = 0; i < batchNum; i++) { timeline1[i] = utils.mkMessage({user: userId, room: roomId}); + if (i !== (batchNum - 1)) { // miss last one + timeline0[i] = utils.mkMessage({user: userId, room: roomId}); + } } - } + }); it("should reconstruct room state", function() { mockStorageApi.setItem(stateKeyName, { @@ -169,22 +209,25 @@ describe("WebStorageStore", function() { expect(storedRoom).not.toBeNull(); // should only get up to the batch num timeline events expect(storedRoom.timeline.length).toEqual(batchNum); + var timeline = timeline0.concat(timeline1); for (i = 0; i < batchNum; i++) { expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual( - timeline0[i] + timeline[timeline.length - 1 - i] ); } }); it("should sync the timeline for 'live' events " + - "(full low batch; 1+bit live batches)", function() { - var i; - var timelineLive = [ - utils.mkMessage({user: userId, room: roomId}), - utils.mkMessage({user: userId, room: roomId}), - utils.mkMessage({user: userId, room: roomId}), - utils.mkMessage({user: userId, room: roomId}) - ]; + "(full hi batch; 1+bit live batches)", function() { + // 1 and a bit events go into _live + var timelineLive = []; + timelineLive.push(utils.mkMessage({user: userId, room: roomId})); + for (i = 0; i < batchNum; i++) { + timelineLive.push( + utils.mkMessage({user: userId, room: roomId}) + ); + } + mockStorageApi.setItem(stateKeyName, { events: stateEventMap, pagination_token: "tok" @@ -211,7 +254,6 @@ describe("WebStorageStore", function() { it("should sync the timeline for 'live' events " + "(no low batch; 1 live batches)", function() { - var i; var timelineLive = []; for (i = 0; i < batchNum; i++) { timelineLive.push( @@ -249,14 +291,14 @@ describe("WebStorageStore", function() { }); mockStorageApi.setItem(prefix + "-5", timeline0); mockStorageApi.setItem(prefix + "-4", timeline1); - + var timeline = timeline0.concat(timeline1); var storedRoom = store.getRoom(roomId); expect(storedRoom).not.toBeNull(); // should only get up to the batch num timeline events expect(storedRoom.timeline.length).toEqual(batchNum); for (i = 0; i < batchNum; i++) { expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual( - timeline0[i] + timeline[timeline.length - 1 - i] ); } });