You've already forked matrix-js-sdk
mirror of
https://github.com/matrix-org/matrix-js-sdk.git
synced 2025-09-01 21:21:58 +03:00
Add first cut webstorage implementation. Add very basic test.
This commit is contained in:
Kegan Dougal
@@ -4,7 +4,6 @@
|
||||
* <pre>
|
||||
* Room data is stored as follows:
|
||||
* room_$ROOMID_timeline_$INDEX : [ Event, Event, Event ]
|
||||
* room_$ROOMID_indexes : {event_id: index}
|
||||
* room_$ROOMID_state : {
|
||||
* pagination_token: <oldState.paginationToken>,
|
||||
* events: {
|
||||
@@ -28,20 +27,19 @@
|
||||
*
|
||||
* Retrieval of earlier messages
|
||||
* -----------------------------
|
||||
* Retrieving earlier messages requires a Room which then finds the earliest
|
||||
* event_id (E) in the timeline for the given Room instance. E is then mapped
|
||||
* to an index I in room_$ROOMID_indexes. I is then retrieved from
|
||||
* The earliest event the Room instance knows about is E. Retrieving earlier
|
||||
* messages requires a Room which has a storageToken defined.
|
||||
* This token maps to the index I where the Room is at. Events are then retrieved from
|
||||
* room_$ROOMID_timeline_{I} and events after E are extracted. If the limit
|
||||
* demands more events, I+1 is retrieved, up until I=max $INDEX where it gives
|
||||
* less than the limit.
|
||||
*
|
||||
* Full Insertion
|
||||
* --------------
|
||||
* Storing a room requires the timeline, indexes and state keys for $ROOMID to
|
||||
* Storing a room requires the timeline and state keys for $ROOMID to
|
||||
* be blown away and completely replaced, which is computationally expensive.
|
||||
* Room.timeline is batched according to the given batch size B. These batches
|
||||
* are then inserted into storage as room_$ROOMID_timeline_$INDEX. Indexes for
|
||||
* the events in each batch are also persisted to room_$ROOMID_indexes. Finally,
|
||||
* are then inserted into storage as room_$ROOMID_timeline_$INDEX. Finally,
|
||||
* the current room state is persisted to room_$ROOMID_state.
|
||||
*
|
||||
* Incremental Insertion
|
||||
@@ -62,7 +60,11 @@
|
||||
* the earliest entries are inserted into the $INDEX (the earliest entries are
|
||||
* inverted in _live, so the earliest entry is at index 0, not len-1) until the
|
||||
* batch == B. Then, the remaining entries in _live are batched to $INDEX-1,
|
||||
* $INDEX-2, and so on. This will result in negative indices.
|
||||
* $INDEX-2, and so on. This will result in negative indices. The easiest way to
|
||||
* visualise this is that the timeline goes from new to old, left to right:
|
||||
* -2 -1 0 1
|
||||
* <--NEW---------------------------------------OLD-->
|
||||
* [a,b,c] [d,e,f] [g,h,i] [j,k,l]
|
||||
*
|
||||
* Purging
|
||||
* -------
|
||||
@@ -77,9 +79,6 @@
|
||||
* room_!foo:bar_timeline_0 : [M1, M2, M3, M4]
|
||||
* room_!foo:bar_timeline_1 : [M5, M6, M7, M8]
|
||||
* room_!foo:bar_timeline_2 : [M9]
|
||||
* room_!foo:bar_indexes : { M1: 0, M2: 0, M3: 0, M4: 0,
|
||||
* M5: 1, M6: 1, M7: 1, M8: 1,
|
||||
* M9: 2 }
|
||||
* room_!foo:bar_state: { ... }
|
||||
*
|
||||
* 5 new messages (N1-5, 1=newest) arrive and are then added: [Incremental Insert]
|
||||
@@ -93,14 +92,14 @@
|
||||
* room_!foo:bar_timeline_-1 : [N2, N3, N4, N5]
|
||||
* room_!foo:bar_timeline_-2 : [N1]
|
||||
* room_!foo:bar_timeline_live: []
|
||||
* room_!foo:bar_indexes : {N1: -2, N2: -1, ...}
|
||||
*
|
||||
* And the room is retrieved with 8 messages: [Room Retrieval]
|
||||
* Room.timeline: [N1, N2, N3, N4, N5, M1, M2, M3]
|
||||
* Room.storageToken: => early_index 0
|
||||
*
|
||||
* 3 earlier messages are requested: [Earlier retrieval]
|
||||
* Use storageToken to find batch index 0. Scan batch for earliest event ID.
|
||||
* earliest event = M3
|
||||
* index = room_!foo:bar_indexes[M3] = 0
|
||||
* events = room_!foo:bar_timeline[0] where event > M3 = [M4]
|
||||
* Too few events, use next index and get 2 more:
|
||||
* events = room_!foo:bar_timeline[1] = [M5, M6, M7, M8] => [M5, M6]
|
||||
@@ -112,6 +111,8 @@
|
||||
*/
|
||||
|
||||
var utils = require("../utils");
|
||||
var Room = require("../models/room");
|
||||
var MatrixEvent = require("../models/event").MatrixEvent;
|
||||
|
||||
/**
|
||||
* Construct a web storage store, capable of storing rooms and users.
|
||||
@@ -161,16 +162,25 @@ WebStorageStore.prototype.setSyncToken = function(token) {
|
||||
* @param {Room} room
|
||||
*/
|
||||
WebStorageStore.prototype.storeRoom = function(room) {
|
||||
initRoomStruct(this.store, room);
|
||||
var serRoom = SerialisedRoom.fromRoom(room, this.batchSize);
|
||||
persist(this.store, serRoom);
|
||||
};
|
||||
|
||||
/**
|
||||
* Retrieve a room from web storage.
|
||||
* @param {string} roomId
|
||||
* @return {null}
|
||||
* @return {?Room}
|
||||
*/
|
||||
WebStorageStore.prototype.getRoom = function(roomId) {
|
||||
return null;
|
||||
// probe if room exists; break early if not. Every room should have state.
|
||||
if (!this.store.getItem(keyName(roomId, "state"))) {
|
||||
return null;
|
||||
}
|
||||
var timelineKeys = getTimelineIndices(this.store, roomId);
|
||||
if (timelineKeys.indexOf("live") !== -1) {
|
||||
this._syncTimeline(roomId, timelineKeys);
|
||||
}
|
||||
return loadRoom(this.store, roomId, this.batchSize);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -216,11 +226,180 @@ WebStorageStore.prototype.scrollback = function(room, limit) {
|
||||
return [];
|
||||
};
|
||||
|
||||
function initRoomStruct(store, roomId) {
|
||||
var prefix = "room_" + roomId;
|
||||
store.setItem(prefix + "_timeline_0", []);
|
||||
store.setItem(prefix + "_indexes", {});
|
||||
store.setItem(prefix + "_state", {});
|
||||
/**
|
||||
* Sync the 'live' timeline, batching live events according to 'batchSize'.
|
||||
* @param {string} roomId The room to sync the timeline.
|
||||
* @param {Array<String>} timelineIndices Optional. The indices in the timeline
|
||||
* if known already.
|
||||
*/
|
||||
WebStorageStore.prototype._syncTimeline = function(roomId, timelineIndices) {
|
||||
timelineIndices = timelineIndices || getTimelineIndices(this.store, roomId);
|
||||
var liveEvents = this.store.getItem(keyName(roomId, "timeline", "live")) || [];
|
||||
|
||||
// get the lowest numbered $INDEX batch
|
||||
var lowestIndex = getLowestIndex(timelineIndices);
|
||||
var lowKey = keyName(roomId, "timeline", lowestIndex);
|
||||
var lowestBatch = this.store.getItem(lowKey) || [];
|
||||
|
||||
// fill up the existing batch first.
|
||||
while (lowestBatch.length < this.batchSize && liveEvents.length > 0) {
|
||||
lowestBatch.unshift(liveEvents.shift());
|
||||
}
|
||||
this.store.setItem(lowKey, lowestBatch);
|
||||
|
||||
// start adding new batches as required
|
||||
var batch = [];
|
||||
while (liveEvents.length > 0) {
|
||||
batch.unshift(liveEvents.shift());
|
||||
if (batch.length === this.batchSize) {
|
||||
// persist the full batch and make another
|
||||
lowestIndex--;
|
||||
lowKey = keyName(roomId, "timeline", lowestIndex);
|
||||
this.store.setItem(lowKey, batch);
|
||||
batch = [];
|
||||
}
|
||||
}
|
||||
// reset live array
|
||||
this.store.setItem(keyName(roomId, "timeline", "live"), []);
|
||||
};
|
||||
|
||||
function SerialisedRoom(roomId) {
|
||||
this.state = {
|
||||
events: {}
|
||||
};
|
||||
this.timeline = {
|
||||
// $INDEX: []
|
||||
};
|
||||
this.roomId = roomId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Room instance into a SerialisedRoom instance which can be stored
|
||||
* in the key value store.
|
||||
* @param {Room} room The matrix room to convert
|
||||
* @param {integer} batchSize The number of events per timeline batch
|
||||
* @return {SerialisedRoom} A serialised room representation of 'room'.
|
||||
*/
|
||||
SerialisedRoom.fromRoom = function(room, batchSize) {
|
||||
var self = new SerialisedRoom(room.roomId);
|
||||
var i, ptr;
|
||||
self.state.pagination_token = room.oldState.paginationToken;
|
||||
// [room_$ROOMID_state] downcast to POJO from MatrixEvent
|
||||
utils.forEach(utils.keys(room.currentState.events), function(eventType) {
|
||||
utils.forEach(utils.keys(room.currentState.events[eventType]), function(skey) {
|
||||
if (!self.state.events[eventType]) {
|
||||
self.state.events[eventType] = {};
|
||||
}
|
||||
self.state.events[eventType][skey] = (
|
||||
room.currentState.events[eventType][skey].event
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// [room_$ROOMID_timeline_$INDEX]
|
||||
if (batchSize > 0) {
|
||||
ptr = 0;
|
||||
while (ptr * batchSize < room.timeline.length) {
|
||||
self.timeline[ptr] = room.timeline.slice(
|
||||
ptr * batchSize, (ptr + 1) * batchSize
|
||||
);
|
||||
self.timeline[ptr] = utils.map(self.timeline[ptr][i], function(me) {
|
||||
// use POJO not MatrixEvent
|
||||
return me.event;
|
||||
});
|
||||
ptr++;
|
||||
}
|
||||
}
|
||||
else { // don't batch
|
||||
self.timeline[0] = utils.map(room.timeline, function(matrixEvent) {
|
||||
return matrixEvent.event;
|
||||
});
|
||||
}
|
||||
return self;
|
||||
};
|
||||
|
||||
function loadRoom(store, roomId, numEvents) {
|
||||
var room = new Room(roomId);
|
||||
// populate state (flatten nested struct to event array)
|
||||
var currentStateMap = store.getItem(keyName(roomId, "state"));
|
||||
var stateEvents = [];
|
||||
utils.forEach(utils.keys(currentStateMap.events), function(eventType) {
|
||||
utils.forEach(utils.keys(currentStateMap.events[eventType]), function(skey) {
|
||||
stateEvents.push(currentStateMap[eventType][skey]);
|
||||
});
|
||||
});
|
||||
// TODO: Fix logic dupe with MatrixClient._processRoomEvents
|
||||
var oldStateEvents = utils.map(
|
||||
utils.deepCopy(stateEvents), function(e) {
|
||||
return new MatrixEvent(e);
|
||||
}
|
||||
);
|
||||
var currentStateEvents = utils.map(stateEvents, function(e) {
|
||||
return new MatrixEvent(e);
|
||||
}
|
||||
);
|
||||
room.oldState.setStateEvents(oldStateEvents);
|
||||
room.currentState.setStateEvents(currentStateEvents);
|
||||
|
||||
// add most recent numEvents
|
||||
var recentEvents = [];
|
||||
var index = getLowestIndex(getTimelineIndices(store, roomId));
|
||||
var i, key, batch;
|
||||
while (recentEvents.length < numEvents) {
|
||||
key = keyName(roomId, "timeline", index);
|
||||
batch = store.getItem(key) || [];
|
||||
if (batch.length === 0) {
|
||||
// nothing left in the store.
|
||||
break;
|
||||
}
|
||||
for (i = 0; i < batch.length; i++) {
|
||||
recentEvents.unshift(new MatrixEvent(batch[i]));
|
||||
}
|
||||
}
|
||||
room.addEventsToTimeline(recentEvents.reverse(), true);
|
||||
room.oldState.paginationToken = currentStateMap.pagination_token;
|
||||
return room;
|
||||
}
|
||||
|
||||
function persist(store, serRoom) {
|
||||
store.setItem(keyName(serRoom.roomId, "state"), serRoom.state);
|
||||
utils.keys(serRoom.timeline, function(index) {
|
||||
store.setItem(
|
||||
keyName(serRoom.roomId, "timeline", index),
|
||||
serRoom.timeline[index]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function getTimelineIndices(store, roomId) {
|
||||
var keys = [];
|
||||
for (var i = 0; i < store.length; i++) {
|
||||
if (store.key(i).indexOf(keyName(roomId, "timeline_")) !== -1) {
|
||||
// e.g. room_$ROOMID_timeline_0 => 0
|
||||
keys.push(
|
||||
store.key(i).replace(keyName(roomId, "timeline_"), "")
|
||||
);
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
function getLowestIndex(timelineIndices) {
|
||||
var lowestIndex = 0;
|
||||
var index;
|
||||
for (var i = 0; i < timelineIndices.length; i++) {
|
||||
index = parseInt(timelineIndices[i]);
|
||||
if (index && index < lowestIndex) {
|
||||
lowestIndex = index;
|
||||
}
|
||||
}
|
||||
return lowestIndex;
|
||||
}
|
||||
|
||||
function keyName(roomId, key, index) {
|
||||
return "room_" + roomId + "_" + key + (
|
||||
index === undefined ? "" : ("_" + index)
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user