From b8a3ee1841f3b06a5c97016d4b75a03ab1456ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:07:59 +0200 Subject: [PATCH 01/64] BasePlatform: Add prototype methods for event indexing. --- src/BasePlatform.js | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/BasePlatform.js b/src/BasePlatform.js index a97c14bf90..7f5df822e4 100644 --- a/src/BasePlatform.js +++ b/src/BasePlatform.js @@ -151,4 +151,44 @@ export default class BasePlatform { async setMinimizeToTrayEnabled(enabled: boolean): void { throw new Error("Unimplemented"); } + + supportsEventIndexing(): boolean { + return false; + } + + async initEventIndex(userId: string): boolean { + throw new Error("Unimplemented"); + } + + async addEventToIndex(ev: {}, profile: {}): void { + throw new Error("Unimplemented"); + } + + indexIsEmpty(): Promise { + throw new Error("Unimplemented"); + } + + async commitLiveEvents(): void { + throw new Error("Unimplemented"); + } + + async searchEventIndex(term: string): Promise<{}> { + throw new Error("Unimplemented"); + } + + async addHistoricEvents(events: [], checkpoint: {} = null, oldCheckpoint: {} = null): Promise { + throw new Error("Unimplemented"); + } + + async addCrawlerCheckpoint(checkpoint: {}): Promise<> { + throw new Error("Unimplemented"); + } + + async removeCrawlerCheckpoint(checkpoint: {}): Promise<> { + throw new Error("Unimplemented"); + } + + async deleteEventIndex(): Promise<> { + throw new Error("Unimplemented"); + } } From 9ce478cb0e29fca8bf0815c7f7a13ef29fe573fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:43:53 +0200 Subject: [PATCH 02/64] MatrixChat: Create an event index and start crawling for events. This patch adds support to create an event index if the clients platform supports it and starts an event crawler. The event crawler goes through the room history of encrypted rooms and eventually indexes the whole room history of such rooms. It does this by first creating crawling checkpoints and storing them inside a database. A checkpoint consists of a room_id, direction and token. After the checkpoints are added the client starts a crawler method in the background. The crawler goes through checkpoints in a round-robin way and uses them to fetch historic room messages using the rooms/roomId/messages API endpoint. Every time messages are fetched a new checkpoint is created that will be stored in the database with the fetched events in an atomic way, the old checkpoint is deleted at the same time as well. --- src/MatrixClientPeg.js | 4 + src/components/structures/MatrixChat.js | 231 ++++++++++++++++++++++++ 2 files changed, 235 insertions(+) diff --git a/src/MatrixClientPeg.js b/src/MatrixClientPeg.js index bebb254afc..5c5ee6e4ec 100644 --- a/src/MatrixClientPeg.js +++ b/src/MatrixClientPeg.js @@ -30,6 +30,7 @@ import {verificationMethods} from 'matrix-js-sdk/lib/crypto'; import MatrixClientBackedSettingsHandler from "./settings/handlers/MatrixClientBackedSettingsHandler"; import * as StorageManager from './utils/StorageManager'; import IdentityAuthClient from './IdentityAuthClient'; +import PlatformPeg from "./PlatformPeg"; interface MatrixClientCreds { homeserverUrl: string, @@ -222,6 +223,9 @@ class MatrixClientPeg { this.matrixClient = createMatrixClient(opts); + const platform = PlatformPeg.get(); + if (platform.supportsEventIndexing()) platform.initEventIndex(creds.userId); + // we're going to add eventlisteners for each matrix event tile, so the // potential number of event listeners is quite high. this.matrixClient.setMaxListeners(500); diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index da67416400..218b7e4d4e 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1262,6 +1262,7 @@ export default createReactClass({ // to do the first sync this.firstSyncComplete = false; this.firstSyncPromise = Promise.defer(); + this.crawlerChekpoints = []; const cli = MatrixClientPeg.get(); const IncomingSasDialog = sdk.getComponent('views.dialogs.IncomingSasDialog'); @@ -1287,6 +1288,75 @@ export default createReactClass({ return self._loggedInView.child.canResetTimelineInRoom(roomId); }); + cli.on('sync', async (state, prevState, data) => { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + if (prevState === null && state === "PREPARED") { + /// Load our stored checkpoints, if any. + self.crawlerChekpoints = await platform.loadCheckpoints(); + console.log("Seshat: Loaded checkpoints", + self.crawlerChekpoints); + return; + } + + if (prevState === "PREPARED" && state === "SYNCING") { + const addInitialCheckpoints = async () => { + const client = MatrixClientPeg.get(); + const rooms = client.getRooms(); + + const isRoomEncrypted = (room) => { + return client.isRoomEncrypted(room.roomId); + }; + + // We only care to crawl the encrypted rooms, non-encrytped + // rooms can use the search provided by the Homeserver. + const encryptedRooms = rooms.filter(isRoomEncrypted); + + console.log("Seshat: Adding initial crawler checkpoints"); + + // Gather the prev_batch tokens and create checkpoints for + // our message crawler. + await Promise.all(encryptedRooms.map(async (room) => { + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + console.log("Seshat: Got token for indexer", + room.roomId, token); + + const backCheckpoint = { + roomId: room.roomId, + token: token, + direction: "b", + }; + + const forwardCheckpoint = { + roomId: room.roomId, + token: token, + direction: "f", + }; + + await platform.addCrawlerCheckpoint(backCheckpoint); + await platform.addCrawlerCheckpoint(forwardCheckpoint); + self.crawlerChekpoints.push(backCheckpoint); + self.crawlerChekpoints.push(forwardCheckpoint); + })); + }; + + // If our indexer is empty we're most likely running Riot the + // first time with indexing support or running it with an + // initial sync. Add checkpoints to crawl our encrypted rooms. + const eventIndexWasEmpty = await platform.isEventIndexEmpty(); + if (eventIndexWasEmpty) await addInitialCheckpoints(); + + // Start our crawler. + const crawlerHandle = {}; + self.crawlerFunc(crawlerHandle); + self.crawlerRef = crawlerHandle; + return; + } + }); + cli.on('sync', function(state, prevState, data) { // LifecycleStore and others cannot directly subscribe to matrix client for // events because flux only allows store state changes during flux dispatches. @@ -1930,4 +2000,165 @@ export default createReactClass({ {view} ; }, + + async crawlerFunc(handle) { + // TODO either put this in a better place or find a library provided + // method that does this. + const sleep = async (ms) => { + return new Promise(resolve => setTimeout(resolve, ms)); + }; + + let cancelled = false; + + console.log("Seshat: Started crawler function"); + + const client = MatrixClientPeg.get(); + const platform = PlatformPeg.get(); + + handle.cancel = () => { + cancelled = true; + }; + + while (!cancelled) { + // This is a low priority task and we don't want to spam our + // Homeserver with /messages requests so we set a hefty 3s timeout + // here. + await sleep(3000); + + if (cancelled) { + break; + } + + const checkpoint = this.crawlerChekpoints.shift(); + + /// There is no checkpoint available currently, one may appear if + // a sync with limited room timelines happens, so go back to sleep. + if (checkpoint === undefined) { + continue; + } + + console.log("Seshat: crawling using checkpoint", checkpoint); + + // We have a checkpoint, let us fetch some messages, again, very + // conservatively to not bother our Homeserver too much. + const eventMapper = client.getEventMapper(); + // TODO we need to ensure to use member lazy loading with this + // request so we get the correct profiles. + const res = await client._createMessagesRequest(checkpoint.roomId, + checkpoint.token, 100, checkpoint.direction); + + if (res.chunk.length === 0) { + // We got to the start/end of our timeline, lets just + // delete our checkpoint and go back to sleep. + await platform.removeCrawlerCheckpoint(checkpoint); + continue; + } + + // Convert the plain JSON events into Matrix events so they get + // decrypted if necessary. + const matrixEvents = res.chunk.map(eventMapper); + const stateEvents = res.state.map(eventMapper); + + const profiles = {}; + + stateEvents.forEach(ev => { + if (ev.event.content && + ev.event.content.membership === "join") { + profiles[ev.event.sender] = { + displayname: ev.event.content.displayname, + avatar_url: ev.event.content.avatar_url, + }; + } + }); + + const decryptionPromises = []; + + matrixEvents.forEach(ev => { + if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { + // TODO the decryption promise is a private property, this + // should either be made public or we should convert the + // event that gets fired when decryption is done into a + // promise using the once event emitter method: + // https://nodejs.org/api/events.html#events_events_once_emitter_name + decryptionPromises.push(ev._decryptionPromise); + } + }); + + // Let us wait for all the events to get decrypted. + await Promise.all(decryptionPromises); + + // We filter out events for which decryption failed, are redacted + // or aren't of a type that we know how to index. + const isValidEvent = (value) => { + return ([ + "m.room.message", + "m.room.name", + "m.room.topic", + ].indexOf(value.getType()) >= 0 + && !value.isRedacted() && !value.isDecryptionFailure() + ); + // TODO do we need to check if the event has all the valid + // attributes? + }; + + // TODO if there ar no events at this point we're missing a lot + // decryption keys, do we wan't to retry this checkpoint at a later + // stage? + const filteredEvents = matrixEvents.filter(isValidEvent); + + // Let us convert the events back into a format that Seshat can + // consume. + const events = filteredEvents.map((ev) => { + const jsonEvent = ev.toJSON(); + + let e; + if (ev.isEncrypted()) e = jsonEvent.decrypted; + else e = jsonEvent; + + let profile = {}; + if (e.sender in profiles) profile = profiles[e.sender]; + const object = { + event: e, + profile: profile, + }; + return object; + }); + + // Create a new checkpoint so we can continue crawling the room for + // messages. + const newCheckpoint = { + roomId: checkpoint.roomId, + token: res.end, + fullCrawl: checkpoint.fullCrawl, + direction: checkpoint.direction, + }; + + console.log( + "Seshat: Crawled room", + client.getRoom(checkpoint.roomId).name, + "and fetched", events.length, "events.", + ); + + try { + const eventsAlreadyAdded = await platform.addHistoricEvents( + events, newCheckpoint, checkpoint); + // If all events were already indexed we assume that we catched + // up with our index and don't need to crawl the room further. + // Let us delete the checkpoint in that case, otherwise push + // the new checkpoint to be used by the crawler. + if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { + await platform.removeCrawlerCheckpoint(newCheckpoint); + } else { + this.crawlerChekpoints.push(newCheckpoint); + } + } catch (e) { + console.log("Seshat: Error durring a crawl", e); + // An error occured, put the checkpoint back so we + // can retry. + this.crawlerChekpoints.push(checkpoint); + } + } + + console.log("Seshat: Stopping crawler function"); + }, }); From b23ba5f8811488c16412b6ebe2d141f1b9e18f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:27:01 +0200 Subject: [PATCH 03/64] MatrixChat: Stop the crawler function and delete the index when logging out. --- src/components/structures/MatrixChat.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 218b7e4d4e..7eda69ad9b 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1221,7 +1221,15 @@ export default createReactClass({ /** * Called when the session is logged out */ - _onLoggedOut: function() { + _onLoggedOut: async function() { + const platform = PlatformPeg.get(); + + if (platform.supportsEventIndexing()) { + console.log("Seshat: Deleting event index."); + this.crawlerRef.cancel(); + await platform.deleteEventIndex(); + } + this.notifyNewScreen('login'); this.setStateForNewView({ view: VIEWS.LOGIN, From 5e7076e985fd95a7978099322457823f96daff8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:28:36 +0200 Subject: [PATCH 04/64] MatrixChat: Add live events to the event index as well. --- src/components/structures/MatrixChat.js | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 7eda69ad9b..5c4db4a562 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1271,6 +1271,7 @@ export default createReactClass({ this.firstSyncComplete = false; this.firstSyncPromise = Promise.defer(); this.crawlerChekpoints = []; + this.liveEventsForIndex = new Set(); const cli = MatrixClientPeg.get(); const IncomingSasDialog = sdk.getComponent('views.dialogs.IncomingSasDialog'); @@ -1363,6 +1364,14 @@ export default createReactClass({ self.crawlerRef = crawlerHandle; return; } + + if (prevState === "SYNCING" && state === "SYNCING") { + // A sync was done, presumably we queued up some live events, + // commit them now. + console.log("Seshat: Committing events"); + await platform.commitLiveEvents(); + return; + } }); cli.on('sync', function(state, prevState, data) { @@ -1447,6 +1456,44 @@ export default createReactClass({ }, null, true); }); + cli.on("Room.timeline", async (ev, room, toStartOfTimeline, removed, data) => { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + // We only index encrypted rooms locally. + if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; + + // If it isn't a live event or if it's redacted there's nothing to + // do. + if (toStartOfTimeline || !data || !data.liveEvent + || ev.isRedacted()) { + return; + } + + // If the event is not yet decrypted mark it for the + // Event.decrypted callback. + if (ev.isBeingDecrypted()) { + const eventId = ev.getId(); + self.liveEventsForIndex.add(eventId); + } else { + // If the event is decrypted or is unencrypted add it to the + // index now. + await self.addLiveEventToIndex(ev); + } + }); + + cli.on("Event.decrypted", async (ev, err) => { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + const eventId = ev.getId(); + + // If the event isn't in our live event set, ignore it. + if (!self.liveEventsForIndex.delete(eventId)) return; + if (err) return; + await self.addLiveEventToIndex(ev); + }); + cli.on("accountData", function(ev) { if (ev.getType() === 'im.vector.web.settings') { if (ev.getContent() && ev.getContent().theme) { @@ -2009,6 +2056,24 @@ export default createReactClass({ ; }, + async addLiveEventToIndex(ev) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + if (["m.room.message", "m.room.name", "m.room.topic"] + .indexOf(ev.getType()) == -1) { + return; + } + + const e = ev.toJSON().decrypted; + const profile = { + displayname: ev.sender.rawDisplayName, + avatar_url: ev.sender.getMxcAvatarUrl(), + }; + + platform.addEventToIndex(e, profile); + }, + async crawlerFunc(handle) { // TODO either put this in a better place or find a library provided // method that does this. From 4acec19d40ba57f789f4c1293ebeb6774babc6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:32:55 +0200 Subject: [PATCH 05/64] MatrixChat: Add new crawler checkpoints if there was a limited timeline. A sync call may not have all events that happened since the last time the client synced. In such a case the room is marked as limited and events need to be fetched separately. When such a sync call happens our event index will have a gap. To close the gap checkpoints are added to start crawling our room again. Unnecessary full re-crawls are prevented by checking if our current /room/roomId/messages request contains only events that were already present in our event index. --- src/components/structures/MatrixChat.js | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 5c4db4a562..d423bbd592 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1281,8 +1281,11 @@ export default createReactClass({ // particularly noticeable when there are lots of 'limited' /sync responses // such as when laptops unsleep. // https://github.com/vector-im/riot-web/issues/3307#issuecomment-282895568 - cli.setCanResetTimelineCallback(function(roomId) { + cli.setCanResetTimelineCallback(async function(roomId) { console.log("Request to reset timeline in room ", roomId, " viewing:", self.state.currentRoomId); + // TODO is there a better place to plug this in + await self.addCheckpointForLimitedRoom(roomId); + if (roomId !== self.state.currentRoomId) { // It is safe to remove events from rooms we are not viewing. return true; @@ -2234,4 +2237,41 @@ export default createReactClass({ console.log("Seshat: Stopping crawler function"); }, + + async addCheckpointForLimitedRoom(roomId) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + if (!MatrixClientPeg.get().isRoomEncrypted(roomId)) return; + + const client = MatrixClientPeg.get(); + const room = client.getRoom(roomId); + + if (room === null) return; + + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + const backwardsCheckpoint = { + roomId: room.roomId, + token: token, + fullCrawl: false, + direction: "b", + }; + + const forwardsCheckpoint = { + roomId: room.roomId, + token: token, + fullCrawl: false, + direction: "f", + }; + + console.log("Seshat: Added checkpoint because of a limited timeline", + backwardsCheckpoint, forwardsCheckpoint); + + await platform.addCrawlerCheckpoint(backwardsCheckpoint); + await platform.addCrawlerCheckpoint(forwardsCheckpoint); + + this.crawlerChekpoints.push(backwardsCheckpoint); + this.crawlerChekpoints.push(forwardsCheckpoint); + }, }); From 3f5369183404be057af45fa248556572804727b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 11 Oct 2019 16:40:10 +0200 Subject: [PATCH 06/64] RoomView: Use platform specific search if our platform supports it. This patch extends our search to include our platform specific event index. There are 3 search scenarios and are handled differently when platform support for indexing is present: - Search a single non-encrypted room: Use the server-side search like before. - Search a single encrypted room: Search using our platform specific event index. - Search across all rooms: Search encrypted rooms using our local event index. Search non-encrypted rooms using the classic server-side search. Combine the results. The combined search will result in having twice the amount of search results since comparing the scores fairly wasn't deemed sensible. --- src/components/structures/RoomView.js | 115 ++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 5 deletions(-) diff --git a/src/components/structures/RoomView.js b/src/components/structures/RoomView.js index 4de573479d..1b44335f51 100644 --- a/src/components/structures/RoomView.js +++ b/src/components/structures/RoomView.js @@ -34,6 +34,7 @@ import { _t } from '../../languageHandler'; import {RoomPermalinkCreator} from '../../utils/permalinks/Permalinks'; import MatrixClientPeg from '../../MatrixClientPeg'; +import PlatformPeg from "../../PlatformPeg"; import ContentMessages from '../../ContentMessages'; import Modal from '../../Modal'; import sdk from '../../index'; @@ -1140,12 +1141,116 @@ module.exports = createReactClass({ } debuglog("sending search request"); + const platform = PlatformPeg.get(); - const searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter: filter, - term: term, - }); - this._handleSearchResult(searchPromise).done(); + if (platform.supportsEventIndexing()) { + const combinedSearchFunc = async (searchTerm) => { + // Create two promises, one for the local search, one for the + // server-side search. + const client = MatrixClientPeg.get(); + const serverSidePromise = client.searchRoomEvents({ + term: searchTerm, + }); + const localPromise = localSearchFunc(searchTerm); + + // Wait for both promises to resolve. + await Promise.all([serverSidePromise, localPromise]); + + // Get both search results. + const localResult = await localPromise; + const serverSideResult = await serverSidePromise; + + // Combine the search results into one result. + const result = {}; + + // Our localResult and serverSideResult are both ordered by + // recency separetly, when we combine them the order might not + // be the right one so we need to sort them. + const compare = (a, b) => { + const aEvent = a.context.getEvent().event; + const bEvent = b.context.getEvent().event; + + if (aEvent.origin_server_ts > + bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < + bEvent.origin_server_ts) return 1; + return 0; + }; + + result.count = localResult.count + serverSideResult.count; + result.results = localResult.results.concat( + serverSideResult.results).sort(compare); + result.highlights = localResult.highlights.concat( + serverSideResult.highlights); + + return result; + }; + + const localSearchFunc = async (searchTerm, roomId = undefined) => { + const searchArgs = { + search_term: searchTerm, + before_limit: 1, + after_limit: 1, + order_by_recency: true, + }; + + if (roomId !== undefined) { + searchArgs.room_id = roomId; + } + + const localResult = await platform.searchEventIndex( + searchArgs); + + const response = { + search_categories: { + room_events: localResult, + }, + }; + + const emptyResult = { + results: [], + highlights: [], + }; + + // TODO is there a better way to convert our result into what + // is expected by the handler method. + const result = MatrixClientPeg.get()._processRoomEventsSearch( + emptyResult, response); + + return result; + }; + + let searchPromise; + + if (scope === "Room") { + const roomId = this.state.room.roomId; + + if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { + // The search is for a single encrypted room, use our local + // search method. + searchPromise = localSearchFunc(term, roomId); + } else { + // The search is for a single non-encrypted room, use the + // server-side search. + searchPromise = MatrixClientPeg.get().searchRoomEvents({ + filter: filter, + term: term, + }); + } + } else { + // Search across all rooms, combine a server side search and a + // local search. + searchPromise = combinedSearchFunc(term); + } + + this._handleSearchResult(searchPromise).done(); + } else { + const searchPromise = MatrixClientPeg.get().searchRoomEvents({ + filter: filter, + term: term, + }); + this._handleSearchResult(searchPromise).done(); + } }, _handleSearchResult: function(searchPromise) { From 1b63886a6baca1a4191f83992609e58e5e6dc43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 18 Oct 2019 16:31:39 +0200 Subject: [PATCH 07/64] MatrixChat: Add more detailed logging to the event crawler. --- src/components/structures/MatrixChat.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index d423bbd592..3558cda586 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -2101,7 +2101,10 @@ export default createReactClass({ // here. await sleep(3000); + console.log("Seshat: Running the crawler loop."); + if (cancelled) { + console.log("Seshat: Cancelling the crawler."); break; } @@ -2124,6 +2127,7 @@ export default createReactClass({ checkpoint.token, 100, checkpoint.direction); if (res.chunk.length === 0) { + console.log("Seshat: Done with the checkpoint", checkpoint) // We got to the start/end of our timeline, lets just // delete our checkpoint and go back to sleep. await platform.removeCrawlerCheckpoint(checkpoint); @@ -2223,6 +2227,7 @@ export default createReactClass({ // Let us delete the checkpoint in that case, otherwise push // the new checkpoint to be used by the crawler. if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { + console.log("Seshat: Checkpoint had already all events added, stopping the crawl", checkpoint); await platform.removeCrawlerCheckpoint(newCheckpoint); } else { this.crawlerChekpoints.push(newCheckpoint); From 89f14e55a2bb31959893f138813957acd957e032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 18 Oct 2019 16:32:43 +0200 Subject: [PATCH 08/64] MatrixChat: Catch errors when fetching room messages in the crawler. --- src/components/structures/MatrixChat.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 3558cda586..2f9e64efa9 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -2123,8 +2123,17 @@ export default createReactClass({ const eventMapper = client.getEventMapper(); // TODO we need to ensure to use member lazy loading with this // request so we get the correct profiles. - const res = await client._createMessagesRequest(checkpoint.roomId, - checkpoint.token, 100, checkpoint.direction); + let res; + + try { + res = await client._createMessagesRequest( + checkpoint.roomId, checkpoint.token, 100, + checkpoint.direction); + } catch (e) { + console.log("Seshat: Error crawling events:", e) + this.crawlerChekpoints.push(checkpoint); + continue + } if (res.chunk.length === 0) { console.log("Seshat: Done with the checkpoint", checkpoint) From 64061173e19507ce40241989a1fb55ac705cd648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 18 Oct 2019 16:33:07 +0200 Subject: [PATCH 09/64] MatrixChat: Check if our state array is empty in the crawled messages response. --- src/components/structures/MatrixChat.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 2f9e64efa9..51cf92da5f 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -2146,7 +2146,10 @@ export default createReactClass({ // Convert the plain JSON events into Matrix events so they get // decrypted if necessary. const matrixEvents = res.chunk.map(eventMapper); - const stateEvents = res.state.map(eventMapper); + let stateEvents = []; + if (res.state !== undefined) { + stateEvents = res.state.map(eventMapper); + } const profiles = {}; From 2c5565e5020edfe0306836fb37d49a8f410df2ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Fri, 18 Oct 2019 16:36:31 +0200 Subject: [PATCH 10/64] MatrixChat: Add some missing semicolons. --- src/components/structures/MatrixChat.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 51cf92da5f..402790df98 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -2130,13 +2130,13 @@ export default createReactClass({ checkpoint.roomId, checkpoint.token, 100, checkpoint.direction); } catch (e) { - console.log("Seshat: Error crawling events:", e) + console.log("Seshat: Error crawling events:", e); this.crawlerChekpoints.push(checkpoint); continue } if (res.chunk.length === 0) { - console.log("Seshat: Done with the checkpoint", checkpoint) + console.log("Seshat: Done with the checkpoint", checkpoint); // We got to the start/end of our timeline, lets just // delete our checkpoint and go back to sleep. await platform.removeCrawlerCheckpoint(checkpoint); @@ -2239,7 +2239,8 @@ export default createReactClass({ // Let us delete the checkpoint in that case, otherwise push // the new checkpoint to be used by the crawler. if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { - console.log("Seshat: Checkpoint had already all events added, stopping the crawl", checkpoint); + console.log("Seshat: Checkpoint had already all events", + "added, stopping the crawl", checkpoint); await platform.removeCrawlerCheckpoint(newCheckpoint); } else { this.crawlerChekpoints.push(newCheckpoint); From cfdcf45ac6eb019242b5d969ce8018fae195caec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 13:29:07 +0100 Subject: [PATCH 11/64] MatrixChat: Move the event indexing logic into separate modules. --- src/EventIndexPeg.js | 74 +++++ src/EventIndexing.js | 404 ++++++++++++++++++++++++ src/Lifecycle.js | 2 + src/MatrixClientPeg.js | 4 +- src/components/structures/MatrixChat.js | 371 ++-------------------- 5 files changed, 499 insertions(+), 356 deletions(-) create mode 100644 src/EventIndexPeg.js create mode 100644 src/EventIndexing.js diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js new file mode 100644 index 0000000000..794450e4b7 --- /dev/null +++ b/src/EventIndexPeg.js @@ -0,0 +1,74 @@ +/* +Copyright 2019 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * Holds the current Platform object used by the code to do anything + * specific to the platform we're running on (eg. web, electron) + * Platforms are provided by the app layer. + * This allows the app layer to set a Platform without necessarily + * having to have a MatrixChat object + */ + +import PlatformPeg from "./PlatformPeg"; +import EventIndex from "./EventIndexing"; +import MatrixClientPeg from "./MatrixClientPeg"; + +class EventIndexPeg { + constructor() { + this.index = null; + } + + /** + * Returns the current Event index object for the application. Can be null + * if the platform doesn't support event indexing. + */ + get() { + return this.index; + } + + /** Create a new EventIndex and initialize it if the platform supports it. + * Returns true if an EventIndex was successfully initialized, false + * otherwise. + */ + async init() { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return false; + + let index = new EventIndex(); + + const userId = MatrixClientPeg.get().getUserId(); + // TODO log errors here and return false if it errors out. + await index.init(userId); + this.index = index; + + return true + } + + async stop() { + if (this.index == null) return; + index.stopCrawler(); + } + + async deleteEventIndex() { + if (this.index == null) return; + index.deleteEventIndex(); + } +} + +if (!global.mxEventIndexPeg) { + global.mxEventIndexPeg = new EventIndexPeg(); +} +module.exports = global.mxEventIndexPeg; diff --git a/src/EventIndexing.js b/src/EventIndexing.js new file mode 100644 index 0000000000..21ee8f3da6 --- /dev/null +++ b/src/EventIndexing.js @@ -0,0 +1,404 @@ +/* +Copyright 2019 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import PlatformPeg from "./PlatformPeg"; +import MatrixClientPeg from "./MatrixClientPeg"; + +/** + * Event indexing class that wraps the platform specific event indexing. + */ +export default class EventIndexer { + constructor() { + this.crawlerChekpoints = []; + // The time that the crawler will wait between /rooms/{room_id}/messages + // requests + this._crawler_timeout = 3000; + this._crawlerRef = null; + this.liveEventsForIndex = new Set(); + } + + async init(userId) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return false; + platform.initEventIndex(userId); + } + + async onSync(state, prevState, data) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + if (prevState === null && state === "PREPARED") { + // Load our stored checkpoints, if any. + this.crawlerChekpoints = await platform.loadCheckpoints(); + console.log("Seshat: Loaded checkpoints", + this.crawlerChekpoints); + return; + } + + if (prevState === "PREPARED" && state === "SYNCING") { + const addInitialCheckpoints = async () => { + const client = MatrixClientPeg.get(); + const rooms = client.getRooms(); + + const isRoomEncrypted = (room) => { + return client.isRoomEncrypted(room.roomId); + }; + + // We only care to crawl the encrypted rooms, non-encrytped + // rooms can use the search provided by the Homeserver. + const encryptedRooms = rooms.filter(isRoomEncrypted); + + console.log("Seshat: Adding initial crawler checkpoints"); + + // Gather the prev_batch tokens and create checkpoints for + // our message crawler. + await Promise.all(encryptedRooms.map(async (room) => { + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + console.log("Seshat: Got token for indexer", + room.roomId, token); + + const backCheckpoint = { + roomId: room.roomId, + token: token, + direction: "b", + }; + + const forwardCheckpoint = { + roomId: room.roomId, + token: token, + direction: "f", + }; + + await platform.addCrawlerCheckpoint(backCheckpoint); + await platform.addCrawlerCheckpoint(forwardCheckpoint); + this.crawlerChekpoints.push(backCheckpoint); + this.crawlerChekpoints.push(forwardCheckpoint); + })); + }; + + // If our indexer is empty we're most likely running Riot the + // first time with indexing support or running it with an + // initial sync. Add checkpoints to crawl our encrypted rooms. + const eventIndexWasEmpty = await platform.isEventIndexEmpty(); + if (eventIndexWasEmpty) await addInitialCheckpoints(); + + // Start our crawler. + this.startCrawler(); + return; + } + + if (prevState === "SYNCING" && state === "SYNCING") { + // A sync was done, presumably we queued up some live events, + // commit them now. + console.log("Seshat: Committing events"); + await platform.commitLiveEvents(); + return; + } + } + + async onRoomTimeline(ev, room, toStartOfTimeline, removed, data) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + // We only index encrypted rooms locally. + if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; + + // If it isn't a live event or if it's redacted there's nothing to + // do. + if (toStartOfTimeline || !data || !data.liveEvent + || ev.isRedacted()) { + return; + } + + // If the event is not yet decrypted mark it for the + // Event.decrypted callback. + if (ev.isBeingDecrypted()) { + const eventId = ev.getId(); + this.liveEventsForIndex.add(eventId); + } else { + // If the event is decrypted or is unencrypted add it to the + // index now. + await this.addLiveEventToIndex(ev); + } + } + + async onEventDecrypted(ev, err) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + const eventId = ev.getId(); + + // If the event isn't in our live event set, ignore it. + if (!this.liveEventsForIndex.delete(eventId)) return; + if (err) return; + await this.addLiveEventToIndex(ev); + } + + async addLiveEventToIndex(ev) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + + if (["m.room.message", "m.room.name", "m.room.topic"] + .indexOf(ev.getType()) == -1) { + return; + } + + const e = ev.toJSON().decrypted; + const profile = { + displayname: ev.sender.rawDisplayName, + avatar_url: ev.sender.getMxcAvatarUrl(), + }; + + platform.addEventToIndex(e, profile); + } + + async crawlerFunc(handle) { + // TODO either put this in a better place or find a library provided + // method that does this. + const sleep = async (ms) => { + return new Promise(resolve => setTimeout(resolve, ms)); + }; + + let cancelled = false; + + console.log("Seshat: Started crawler function"); + + const client = MatrixClientPeg.get(); + const platform = PlatformPeg.get(); + + handle.cancel = () => { + cancelled = true; + }; + + while (!cancelled) { + // This is a low priority task and we don't want to spam our + // Homeserver with /messages requests so we set a hefty timeout + // here. + await sleep(this._crawler_timeout); + + console.log("Seshat: Running the crawler loop."); + + if (cancelled) { + console.log("Seshat: Cancelling the crawler."); + break; + } + + const checkpoint = this.crawlerChekpoints.shift(); + + /// There is no checkpoint available currently, one may appear if + // a sync with limited room timelines happens, so go back to sleep. + if (checkpoint === undefined) { + continue; + } + + console.log("Seshat: crawling using checkpoint", checkpoint); + + // We have a checkpoint, let us fetch some messages, again, very + // conservatively to not bother our Homeserver too much. + const eventMapper = client.getEventMapper(); + // TODO we need to ensure to use member lazy loading with this + // request so we get the correct profiles. + let res; + + try { + res = await client._createMessagesRequest( + checkpoint.roomId, checkpoint.token, 100, + checkpoint.direction); + } catch (e) { + console.log("Seshat: Error crawling events:", e); + this.crawlerChekpoints.push(checkpoint); + continue + } + + if (res.chunk.length === 0) { + console.log("Seshat: Done with the checkpoint", checkpoint); + // We got to the start/end of our timeline, lets just + // delete our checkpoint and go back to sleep. + await platform.removeCrawlerCheckpoint(checkpoint); + continue; + } + + // Convert the plain JSON events into Matrix events so they get + // decrypted if necessary. + const matrixEvents = res.chunk.map(eventMapper); + let stateEvents = []; + if (res.state !== undefined) { + stateEvents = res.state.map(eventMapper); + } + + const profiles = {}; + + stateEvents.forEach(ev => { + if (ev.event.content && + ev.event.content.membership === "join") { + profiles[ev.event.sender] = { + displayname: ev.event.content.displayname, + avatar_url: ev.event.content.avatar_url, + }; + } + }); + + const decryptionPromises = []; + + matrixEvents.forEach(ev => { + if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { + // TODO the decryption promise is a private property, this + // should either be made public or we should convert the + // event that gets fired when decryption is done into a + // promise using the once event emitter method: + // https://nodejs.org/api/events.html#events_events_once_emitter_name + decryptionPromises.push(ev._decryptionPromise); + } + }); + + // Let us wait for all the events to get decrypted. + await Promise.all(decryptionPromises); + + // We filter out events for which decryption failed, are redacted + // or aren't of a type that we know how to index. + const isValidEvent = (value) => { + return ([ + "m.room.message", + "m.room.name", + "m.room.topic", + ].indexOf(value.getType()) >= 0 + && !value.isRedacted() && !value.isDecryptionFailure() + ); + // TODO do we need to check if the event has all the valid + // attributes? + }; + + // TODO if there ar no events at this point we're missing a lot + // decryption keys, do we wan't to retry this checkpoint at a later + // stage? + const filteredEvents = matrixEvents.filter(isValidEvent); + + // Let us convert the events back into a format that Seshat can + // consume. + const events = filteredEvents.map((ev) => { + const jsonEvent = ev.toJSON(); + + let e; + if (ev.isEncrypted()) e = jsonEvent.decrypted; + else e = jsonEvent; + + let profile = {}; + if (e.sender in profiles) profile = profiles[e.sender]; + const object = { + event: e, + profile: profile, + }; + return object; + }); + + // Create a new checkpoint so we can continue crawling the room for + // messages. + const newCheckpoint = { + roomId: checkpoint.roomId, + token: res.end, + fullCrawl: checkpoint.fullCrawl, + direction: checkpoint.direction, + }; + + console.log( + "Seshat: Crawled room", + client.getRoom(checkpoint.roomId).name, + "and fetched", events.length, "events.", + ); + + try { + const eventsAlreadyAdded = await platform.addHistoricEvents( + events, newCheckpoint, checkpoint); + // If all events were already indexed we assume that we catched + // up with our index and don't need to crawl the room further. + // Let us delete the checkpoint in that case, otherwise push + // the new checkpoint to be used by the crawler. + if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { + console.log("Seshat: Checkpoint had already all events", + "added, stopping the crawl", checkpoint); + await platform.removeCrawlerCheckpoint(newCheckpoint); + } else { + this.crawlerChekpoints.push(newCheckpoint); + } + } catch (e) { + console.log("Seshat: Error durring a crawl", e); + // An error occured, put the checkpoint back so we + // can retry. + this.crawlerChekpoints.push(checkpoint); + } + } + + console.log("Seshat: Stopping crawler function"); + } + + async addCheckpointForLimitedRoom(roomId) { + const platform = PlatformPeg.get(); + if (!platform.supportsEventIndexing()) return; + if (!MatrixClientPeg.get().isRoomEncrypted(roomId)) return; + + const client = MatrixClientPeg.get(); + const room = client.getRoom(roomId); + + if (room === null) return; + + const timeline = room.getLiveTimeline(); + const token = timeline.getPaginationToken("b"); + + const backwardsCheckpoint = { + roomId: room.roomId, + token: token, + fullCrawl: false, + direction: "b", + }; + + const forwardsCheckpoint = { + roomId: room.roomId, + token: token, + fullCrawl: false, + direction: "f", + }; + + console.log("Seshat: Added checkpoint because of a limited timeline", + backwardsCheckpoint, forwardsCheckpoint); + + await platform.addCrawlerCheckpoint(backwardsCheckpoint); + await platform.addCrawlerCheckpoint(forwardsCheckpoint); + + this.crawlerChekpoints.push(backwardsCheckpoint); + this.crawlerChekpoints.push(forwardsCheckpoint); + } + + async deleteEventIndex() { + if (platform.supportsEventIndexing()) { + console.log("Seshat: Deleting event index."); + this.crawlerRef.cancel(); + await platform.deleteEventIndex(); + } + } + + startCrawler() { + const crawlerHandle = {}; + this.crawlerFunc(crawlerHandle); + this.crawlerRef = crawlerHandle; + } + + stopCrawler() { + this._crawlerRef.cancel(); + this._crawlerRef = null; + } +} diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 7490c5d464..0b44f2ed84 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -20,6 +20,7 @@ import Promise from 'bluebird'; import Matrix from 'matrix-js-sdk'; import MatrixClientPeg from './MatrixClientPeg'; +import EventIndexPeg from './EventIndexPeg'; import createMatrixClient from './utils/createMatrixClient'; import Analytics from './Analytics'; import Notifier from './Notifier'; @@ -587,6 +588,7 @@ async function startMatrixClient(startSyncing=true) { if (startSyncing) { await MatrixClientPeg.start(); + await EventIndexPeg.init(); } else { console.warn("Caller requested only auxiliary services be started"); await MatrixClientPeg.assign(); diff --git a/src/MatrixClientPeg.js b/src/MatrixClientPeg.js index 5c5ee6e4ec..6c5b465bb0 100644 --- a/src/MatrixClientPeg.js +++ b/src/MatrixClientPeg.js @@ -31,6 +31,7 @@ import MatrixClientBackedSettingsHandler from "./settings/handlers/MatrixClientB import * as StorageManager from './utils/StorageManager'; import IdentityAuthClient from './IdentityAuthClient'; import PlatformPeg from "./PlatformPeg"; +import EventIndexPeg from "./EventIndexPeg"; interface MatrixClientCreds { homeserverUrl: string, @@ -223,9 +224,6 @@ class MatrixClientPeg { this.matrixClient = createMatrixClient(opts); - const platform = PlatformPeg.get(); - if (platform.supportsEventIndexing()) platform.initEventIndex(creds.userId); - // we're going to add eventlisteners for each matrix event tile, so the // potential number of event listeners is quite high. this.matrixClient.setMaxListeners(500); diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 402790df98..d006247151 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -31,6 +31,7 @@ import Analytics from "../../Analytics"; import { DecryptionFailureTracker } from "../../DecryptionFailureTracker"; import MatrixClientPeg from "../../MatrixClientPeg"; import PlatformPeg from "../../PlatformPeg"; +import EventIndexPeg from "../../EventIndexPeg"; import SdkConfig from "../../SdkConfig"; import * as RoomListSorter from "../../RoomListSorter"; import dis from "../../dispatcher"; @@ -1224,12 +1225,6 @@ export default createReactClass({ _onLoggedOut: async function() { const platform = PlatformPeg.get(); - if (platform.supportsEventIndexing()) { - console.log("Seshat: Deleting event index."); - this.crawlerRef.cancel(); - await platform.deleteEventIndex(); - } - this.notifyNewScreen('login'); this.setStateForNewView({ view: VIEWS.LOGIN, @@ -1270,8 +1265,6 @@ export default createReactClass({ // to do the first sync this.firstSyncComplete = false; this.firstSyncPromise = Promise.defer(); - this.crawlerChekpoints = []; - this.liveEventsForIndex = new Set(); const cli = MatrixClientPeg.get(); const IncomingSasDialog = sdk.getComponent('views.dialogs.IncomingSasDialog'); @@ -1284,7 +1277,10 @@ export default createReactClass({ cli.setCanResetTimelineCallback(async function(roomId) { console.log("Request to reset timeline in room ", roomId, " viewing:", self.state.currentRoomId); // TODO is there a better place to plug this in - await self.addCheckpointForLimitedRoom(roomId); + const eventIndex = EventIndexPeg.get(); + if (eventIndex !== null) { + await eventIndex.addCheckpointForLimitedRoom(roomId); + } if (roomId !== self.state.currentRoomId) { // It is safe to remove events from rooms we are not viewing. @@ -1301,80 +1297,21 @@ export default createReactClass({ }); cli.on('sync', async (state, prevState, data) => { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const eventIndex = EventIndexPeg.get(); + if (eventIndex === null) return; + await eventIndex.onSync(state, prevState, data); + }); - if (prevState === null && state === "PREPARED") { - /// Load our stored checkpoints, if any. - self.crawlerChekpoints = await platform.loadCheckpoints(); - console.log("Seshat: Loaded checkpoints", - self.crawlerChekpoints); - return; - } + cli.on("Room.timeline", async (ev, room, toStartOfTimeline, removed, data) => { + const eventIndex = EventIndexPeg.get(); + if (eventIndex === null) return; + await eventIndex.onRoomTimeline(ev, room, toStartOfTimeline, removed, data); + }); - if (prevState === "PREPARED" && state === "SYNCING") { - const addInitialCheckpoints = async () => { - const client = MatrixClientPeg.get(); - const rooms = client.getRooms(); - - const isRoomEncrypted = (room) => { - return client.isRoomEncrypted(room.roomId); - }; - - // We only care to crawl the encrypted rooms, non-encrytped - // rooms can use the search provided by the Homeserver. - const encryptedRooms = rooms.filter(isRoomEncrypted); - - console.log("Seshat: Adding initial crawler checkpoints"); - - // Gather the prev_batch tokens and create checkpoints for - // our message crawler. - await Promise.all(encryptedRooms.map(async (room) => { - const timeline = room.getLiveTimeline(); - const token = timeline.getPaginationToken("b"); - - console.log("Seshat: Got token for indexer", - room.roomId, token); - - const backCheckpoint = { - roomId: room.roomId, - token: token, - direction: "b", - }; - - const forwardCheckpoint = { - roomId: room.roomId, - token: token, - direction: "f", - }; - - await platform.addCrawlerCheckpoint(backCheckpoint); - await platform.addCrawlerCheckpoint(forwardCheckpoint); - self.crawlerChekpoints.push(backCheckpoint); - self.crawlerChekpoints.push(forwardCheckpoint); - })); - }; - - // If our indexer is empty we're most likely running Riot the - // first time with indexing support or running it with an - // initial sync. Add checkpoints to crawl our encrypted rooms. - const eventIndexWasEmpty = await platform.isEventIndexEmpty(); - if (eventIndexWasEmpty) await addInitialCheckpoints(); - - // Start our crawler. - const crawlerHandle = {}; - self.crawlerFunc(crawlerHandle); - self.crawlerRef = crawlerHandle; - return; - } - - if (prevState === "SYNCING" && state === "SYNCING") { - // A sync was done, presumably we queued up some live events, - // commit them now. - console.log("Seshat: Committing events"); - await platform.commitLiveEvents(); - return; - } + cli.on("Event.decrypted", async (ev, err) => { + const eventIndex = EventIndexPeg.get(); + if (eventIndex === null) return; + await eventIndex.onEventDecrypted(ev, err); }); cli.on('sync', function(state, prevState, data) { @@ -1459,44 +1396,6 @@ export default createReactClass({ }, null, true); }); - cli.on("Room.timeline", async (ev, room, toStartOfTimeline, removed, data) => { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; - - // We only index encrypted rooms locally. - if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; - - // If it isn't a live event or if it's redacted there's nothing to - // do. - if (toStartOfTimeline || !data || !data.liveEvent - || ev.isRedacted()) { - return; - } - - // If the event is not yet decrypted mark it for the - // Event.decrypted callback. - if (ev.isBeingDecrypted()) { - const eventId = ev.getId(); - self.liveEventsForIndex.add(eventId); - } else { - // If the event is decrypted or is unencrypted add it to the - // index now. - await self.addLiveEventToIndex(ev); - } - }); - - cli.on("Event.decrypted", async (ev, err) => { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; - - const eventId = ev.getId(); - - // If the event isn't in our live event set, ignore it. - if (!self.liveEventsForIndex.delete(eventId)) return; - if (err) return; - await self.addLiveEventToIndex(ev); - }); - cli.on("accountData", function(ev) { if (ev.getType() === 'im.vector.web.settings') { if (ev.getContent() && ev.getContent().theme) { @@ -2058,238 +1957,4 @@ export default createReactClass({ {view} ; }, - - async addLiveEventToIndex(ev) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; - - if (["m.room.message", "m.room.name", "m.room.topic"] - .indexOf(ev.getType()) == -1) { - return; - } - - const e = ev.toJSON().decrypted; - const profile = { - displayname: ev.sender.rawDisplayName, - avatar_url: ev.sender.getMxcAvatarUrl(), - }; - - platform.addEventToIndex(e, profile); - }, - - async crawlerFunc(handle) { - // TODO either put this in a better place or find a library provided - // method that does this. - const sleep = async (ms) => { - return new Promise(resolve => setTimeout(resolve, ms)); - }; - - let cancelled = false; - - console.log("Seshat: Started crawler function"); - - const client = MatrixClientPeg.get(); - const platform = PlatformPeg.get(); - - handle.cancel = () => { - cancelled = true; - }; - - while (!cancelled) { - // This is a low priority task and we don't want to spam our - // Homeserver with /messages requests so we set a hefty 3s timeout - // here. - await sleep(3000); - - console.log("Seshat: Running the crawler loop."); - - if (cancelled) { - console.log("Seshat: Cancelling the crawler."); - break; - } - - const checkpoint = this.crawlerChekpoints.shift(); - - /// There is no checkpoint available currently, one may appear if - // a sync with limited room timelines happens, so go back to sleep. - if (checkpoint === undefined) { - continue; - } - - console.log("Seshat: crawling using checkpoint", checkpoint); - - // We have a checkpoint, let us fetch some messages, again, very - // conservatively to not bother our Homeserver too much. - const eventMapper = client.getEventMapper(); - // TODO we need to ensure to use member lazy loading with this - // request so we get the correct profiles. - let res; - - try { - res = await client._createMessagesRequest( - checkpoint.roomId, checkpoint.token, 100, - checkpoint.direction); - } catch (e) { - console.log("Seshat: Error crawling events:", e); - this.crawlerChekpoints.push(checkpoint); - continue - } - - if (res.chunk.length === 0) { - console.log("Seshat: Done with the checkpoint", checkpoint); - // We got to the start/end of our timeline, lets just - // delete our checkpoint and go back to sleep. - await platform.removeCrawlerCheckpoint(checkpoint); - continue; - } - - // Convert the plain JSON events into Matrix events so they get - // decrypted if necessary. - const matrixEvents = res.chunk.map(eventMapper); - let stateEvents = []; - if (res.state !== undefined) { - stateEvents = res.state.map(eventMapper); - } - - const profiles = {}; - - stateEvents.forEach(ev => { - if (ev.event.content && - ev.event.content.membership === "join") { - profiles[ev.event.sender] = { - displayname: ev.event.content.displayname, - avatar_url: ev.event.content.avatar_url, - }; - } - }); - - const decryptionPromises = []; - - matrixEvents.forEach(ev => { - if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) { - // TODO the decryption promise is a private property, this - // should either be made public or we should convert the - // event that gets fired when decryption is done into a - // promise using the once event emitter method: - // https://nodejs.org/api/events.html#events_events_once_emitter_name - decryptionPromises.push(ev._decryptionPromise); - } - }); - - // Let us wait for all the events to get decrypted. - await Promise.all(decryptionPromises); - - // We filter out events for which decryption failed, are redacted - // or aren't of a type that we know how to index. - const isValidEvent = (value) => { - return ([ - "m.room.message", - "m.room.name", - "m.room.topic", - ].indexOf(value.getType()) >= 0 - && !value.isRedacted() && !value.isDecryptionFailure() - ); - // TODO do we need to check if the event has all the valid - // attributes? - }; - - // TODO if there ar no events at this point we're missing a lot - // decryption keys, do we wan't to retry this checkpoint at a later - // stage? - const filteredEvents = matrixEvents.filter(isValidEvent); - - // Let us convert the events back into a format that Seshat can - // consume. - const events = filteredEvents.map((ev) => { - const jsonEvent = ev.toJSON(); - - let e; - if (ev.isEncrypted()) e = jsonEvent.decrypted; - else e = jsonEvent; - - let profile = {}; - if (e.sender in profiles) profile = profiles[e.sender]; - const object = { - event: e, - profile: profile, - }; - return object; - }); - - // Create a new checkpoint so we can continue crawling the room for - // messages. - const newCheckpoint = { - roomId: checkpoint.roomId, - token: res.end, - fullCrawl: checkpoint.fullCrawl, - direction: checkpoint.direction, - }; - - console.log( - "Seshat: Crawled room", - client.getRoom(checkpoint.roomId).name, - "and fetched", events.length, "events.", - ); - - try { - const eventsAlreadyAdded = await platform.addHistoricEvents( - events, newCheckpoint, checkpoint); - // If all events were already indexed we assume that we catched - // up with our index and don't need to crawl the room further. - // Let us delete the checkpoint in that case, otherwise push - // the new checkpoint to be used by the crawler. - if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { - console.log("Seshat: Checkpoint had already all events", - "added, stopping the crawl", checkpoint); - await platform.removeCrawlerCheckpoint(newCheckpoint); - } else { - this.crawlerChekpoints.push(newCheckpoint); - } - } catch (e) { - console.log("Seshat: Error durring a crawl", e); - // An error occured, put the checkpoint back so we - // can retry. - this.crawlerChekpoints.push(checkpoint); - } - } - - console.log("Seshat: Stopping crawler function"); - }, - - async addCheckpointForLimitedRoom(roomId) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; - if (!MatrixClientPeg.get().isRoomEncrypted(roomId)) return; - - const client = MatrixClientPeg.get(); - const room = client.getRoom(roomId); - - if (room === null) return; - - const timeline = room.getLiveTimeline(); - const token = timeline.getPaginationToken("b"); - - const backwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "b", - }; - - const forwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "f", - }; - - console.log("Seshat: Added checkpoint because of a limited timeline", - backwardsCheckpoint, forwardsCheckpoint); - - await platform.addCrawlerCheckpoint(backwardsCheckpoint); - await platform.addCrawlerCheckpoint(forwardsCheckpoint); - - this.crawlerChekpoints.push(backwardsCheckpoint); - this.crawlerChekpoints.push(forwardsCheckpoint); - }, }); From e296fd05c0048e95a98c8777209ecb2990d787f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 15:39:26 +0100 Subject: [PATCH 12/64] RoomView: Move the search logic into a separate module. --- src/EventIndexing.js | 5 + src/Searching.js | 137 ++++++++++++++++++++++++++ src/components/structures/RoomView.js | 125 +---------------------- 3 files changed, 147 insertions(+), 120 deletions(-) create mode 100644 src/Searching.js diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 21ee8f3da6..29f9c48842 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -401,4 +401,9 @@ export default class EventIndexer { this._crawlerRef.cancel(); this._crawlerRef = null; } + + async search(searchArgs) { + const platform = PlatformPeg.get(); + return platform.searchEventIndex(searchArgs) + } } diff --git a/src/Searching.js b/src/Searching.js new file mode 100644 index 0000000000..cd06d9bc67 --- /dev/null +++ b/src/Searching.js @@ -0,0 +1,137 @@ +/* +Copyright 2019 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import EventIndexPeg from "./EventIndexPeg"; +import MatrixClientPeg from "./MatrixClientPeg"; + +function serverSideSearch(term, roomId = undefined) { + let filter; + if (roomId !== undefined) { + filter = { + // XXX: it's unintuitive that the filter for searching doesn't have the same shape as the v2 filter API :( + rooms: [roomId], + }; + } + + let searchPromise = MatrixClientPeg.get().searchRoomEvents({ + filter: filter, + term: term, + }); + + return searchPromise; +} + +function eventIndexSearch(term, roomId = undefined) { + const combinedSearchFunc = async (searchTerm) => { + // Create two promises, one for the local search, one for the + // server-side search. + const client = MatrixClientPeg.get(); + const serverSidePromise = serverSideSearch(searchTerm); + const localPromise = localSearchFunc(searchTerm); + + // Wait for both promises to resolve. + await Promise.all([serverSidePromise, localPromise]); + + // Get both search results. + const localResult = await localPromise; + const serverSideResult = await serverSidePromise; + + // Combine the search results into one result. + const result = {}; + + // Our localResult and serverSideResult are both ordered by + // recency separetly, when we combine them the order might not + // be the right one so we need to sort them. + const compare = (a, b) => { + const aEvent = a.context.getEvent().event; + const bEvent = b.context.getEvent().event; + + if (aEvent.origin_server_ts > + bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < + bEvent.origin_server_ts) return 1; + return 0; + }; + + result.count = localResult.count + serverSideResult.count; + result.results = localResult.results.concat( + serverSideResult.results).sort(compare); + result.highlights = localResult.highlights.concat( + serverSideResult.highlights); + + return result; + }; + + const localSearchFunc = async (searchTerm, roomId = undefined) => { + const searchArgs = { + search_term: searchTerm, + before_limit: 1, + after_limit: 1, + order_by_recency: true, + }; + + if (roomId !== undefined) { + searchArgs.room_id = roomId; + } + + const eventIndex = EventIndexPeg.get(); + + const localResult = await eventIndex.search(searchArgs); + + const response = { + search_categories: { + room_events: localResult, + }, + }; + + const emptyResult = { + results: [], + highlights: [], + }; + + const result = MatrixClientPeg.get()._processRoomEventsSearch( + emptyResult, response); + + return result; + }; + + let searchPromise; + + if (roomId !== undefined) { + if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { + // The search is for a single encrypted room, use our local + // search method. + searchPromise = localSearchFunc(term, roomId); + } else { + // The search is for a single non-encrypted room, use the + // server-side search. + searchPromise = serverSideSearch(term, roomId); + } + } else { + // Search across all rooms, combine a server side search and a + // local search. + searchPromise = combinedSearchFunc(term); + } + + return searchPromise +} + +export default function eventSearch(term, roomId = undefined) { + const eventIndex = EventIndexPeg.get(); + + if (eventIndex === null) return serverSideSearch(term, roomId); + else return eventIndexSearch(term, roomId); +} diff --git a/src/components/structures/RoomView.js b/src/components/structures/RoomView.js index 1b44335f51..9fe54ad164 100644 --- a/src/components/structures/RoomView.js +++ b/src/components/structures/RoomView.js @@ -34,7 +34,6 @@ import { _t } from '../../languageHandler'; import {RoomPermalinkCreator} from '../../utils/permalinks/Permalinks'; import MatrixClientPeg from '../../MatrixClientPeg'; -import PlatformPeg from "../../PlatformPeg"; import ContentMessages from '../../ContentMessages'; import Modal from '../../Modal'; import sdk from '../../index'; @@ -44,6 +43,7 @@ import Tinter from '../../Tinter'; import rate_limited_func from '../../ratelimitedfunc'; import ObjectUtils from '../../ObjectUtils'; import * as Rooms from '../../Rooms'; +import eventSearch from '../../Searching'; import { KeyCode, isOnlyCtrlOrCmdKeyEvent } from '../../Keyboard'; @@ -1130,127 +1130,12 @@ module.exports = createReactClass({ // todo: should cancel any previous search requests. this.searchId = new Date().getTime(); - let filter; - if (scope === "Room") { - filter = { - // XXX: it's unintuitive that the filter for searching doesn't have the same shape as the v2 filter API :( - rooms: [ - this.state.room.roomId, - ], - }; - } + let roomId; + if (scope === "Room") roomId = this.state.room.roomId, debuglog("sending search request"); - const platform = PlatformPeg.get(); - - if (platform.supportsEventIndexing()) { - const combinedSearchFunc = async (searchTerm) => { - // Create two promises, one for the local search, one for the - // server-side search. - const client = MatrixClientPeg.get(); - const serverSidePromise = client.searchRoomEvents({ - term: searchTerm, - }); - const localPromise = localSearchFunc(searchTerm); - - // Wait for both promises to resolve. - await Promise.all([serverSidePromise, localPromise]); - - // Get both search results. - const localResult = await localPromise; - const serverSideResult = await serverSidePromise; - - // Combine the search results into one result. - const result = {}; - - // Our localResult and serverSideResult are both ordered by - // recency separetly, when we combine them the order might not - // be the right one so we need to sort them. - const compare = (a, b) => { - const aEvent = a.context.getEvent().event; - const bEvent = b.context.getEvent().event; - - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; - return 0; - }; - - result.count = localResult.count + serverSideResult.count; - result.results = localResult.results.concat( - serverSideResult.results).sort(compare); - result.highlights = localResult.highlights.concat( - serverSideResult.highlights); - - return result; - }; - - const localSearchFunc = async (searchTerm, roomId = undefined) => { - const searchArgs = { - search_term: searchTerm, - before_limit: 1, - after_limit: 1, - order_by_recency: true, - }; - - if (roomId !== undefined) { - searchArgs.room_id = roomId; - } - - const localResult = await platform.searchEventIndex( - searchArgs); - - const response = { - search_categories: { - room_events: localResult, - }, - }; - - const emptyResult = { - results: [], - highlights: [], - }; - - // TODO is there a better way to convert our result into what - // is expected by the handler method. - const result = MatrixClientPeg.get()._processRoomEventsSearch( - emptyResult, response); - - return result; - }; - - let searchPromise; - - if (scope === "Room") { - const roomId = this.state.room.roomId; - - if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { - // The search is for a single encrypted room, use our local - // search method. - searchPromise = localSearchFunc(term, roomId); - } else { - // The search is for a single non-encrypted room, use the - // server-side search. - searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter: filter, - term: term, - }); - } - } else { - // Search across all rooms, combine a server side search and a - // local search. - searchPromise = combinedSearchFunc(term); - } - - this._handleSearchResult(searchPromise).done(); - } else { - const searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter: filter, - term: term, - }); - this._handleSearchResult(searchPromise).done(); - } + const searchPromise = eventSearch(term, roomId); + this._handleSearchResult(searchPromise).done(); }, _handleSearchResult: function(searchPromise) { From d911055f5d8016ebd2f036d68a9c1ee3f7343af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 15:39:54 +0100 Subject: [PATCH 13/64] MatrixChat: Move the indexing limited room logic to a different event. --- src/components/structures/MatrixChat.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index d006247151..0d3d5abd55 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1276,11 +1276,6 @@ export default createReactClass({ // https://github.com/vector-im/riot-web/issues/3307#issuecomment-282895568 cli.setCanResetTimelineCallback(async function(roomId) { console.log("Request to reset timeline in room ", roomId, " viewing:", self.state.currentRoomId); - // TODO is there a better place to plug this in - const eventIndex = EventIndexPeg.get(); - if (eventIndex !== null) { - await eventIndex.addCheckpointForLimitedRoom(roomId); - } if (roomId !== self.state.currentRoomId) { // It is safe to remove events from rooms we are not viewing. @@ -1314,6 +1309,13 @@ export default createReactClass({ await eventIndex.onEventDecrypted(ev, err); }); + cli.on("Room.timelineReset", async (room, timelineSet, resetAllTimelines) => { + const eventIndex = EventIndexPeg.get(); + if (eventIndex === null) return; + if (resetAllTimelines === true) return; + await eventIndex.addCheckpointForLimitedRoom(roomId); + }); + cli.on('sync', function(state, prevState, data) { // LifecycleStore and others cannot directly subscribe to matrix client for // events because flux only allows store state changes during flux dispatches. From ecbc47c5488bf60b5cc068b09d4a51672b9a5c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 15:40:49 +0100 Subject: [PATCH 14/64] EventIndexing: Rename the stop method. --- src/EventIndexPeg.js | 9 +++++---- src/EventIndexing.js | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 794450e4b7..86fb889c7a 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -57,13 +57,14 @@ class EventIndexPeg { return true } - async stop() { - if (this.index == null) return; - index.stopCrawler(); + stop() { + if (this.index === null) return; + index.stop(); + this.index = null; } async deleteEventIndex() { - if (this.index == null) return; + if (this.index === null) return; index.deleteEventIndex(); } } diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 29f9c48842..92a3a5a1f8 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -34,6 +34,7 @@ export default class EventIndexer { const platform = PlatformPeg.get(); if (!platform.supportsEventIndexing()) return false; platform.initEventIndex(userId); + return true; } async onSync(state, prevState, data) { @@ -397,7 +398,7 @@ export default class EventIndexer { this.crawlerRef = crawlerHandle; } - stopCrawler() { + stop() { this._crawlerRef.cancel(); this._crawlerRef = null; } From d69eb78b661764e9241d14a0c08dff23906245c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 15:41:14 +0100 Subject: [PATCH 15/64] EventIndexing: Add a missing platform getting. --- src/EventIndexing.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 92a3a5a1f8..ebd2ffe983 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -385,6 +385,7 @@ export default class EventIndexer { } async deleteEventIndex() { + const platform = PlatformPeg.get(); if (platform.supportsEventIndexing()) { console.log("Seshat: Deleting event index."); this.crawlerRef.cancel(); From 3502454c615f1a7bc74588f3512661278604d2d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 12 Nov 2019 15:58:38 +0100 Subject: [PATCH 16/64] LifeCycle: Stop the crawler and delete the index when whe log out. --- src/Lifecycle.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 0b44f2ed84..7360cd3231 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -613,6 +613,7 @@ export function onLoggedOut() { // so that React components unmount first. This avoids React soft crashes // that can occur when components try to use a null client. dis.dispatch({action: 'on_logged_out'}); + EventIndexPeg.deleteEventIndex().done(); stopMatrixClient(); _clearStorage().done(); } @@ -648,6 +649,7 @@ export function stopMatrixClient(unsetClient=true) { ActiveWidgetStore.stop(); IntegrationManagers.sharedInstance().stopWatching(); if (DMRoomMap.shared()) DMRoomMap.shared().stop(); + EventIndexPeg.stop(); const cli = MatrixClientPeg.get(); if (cli) { cli.stopClient(); From 008554463d0478e9b06f0da35dce1af83d08eb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 09:52:59 +0100 Subject: [PATCH 17/64] Lifecycle: Move the event index deletion into the clear storage method. --- src/Lifecycle.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 7360cd3231..1e68bcc062 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -613,7 +613,6 @@ export function onLoggedOut() { // so that React components unmount first. This avoids React soft crashes // that can occur when components try to use a null client. dis.dispatch({action: 'on_logged_out'}); - EventIndexPeg.deleteEventIndex().done(); stopMatrixClient(); _clearStorage().done(); } @@ -633,7 +632,13 @@ function _clearStorage() { // we'll never make any requests, so can pass a bogus HS URL baseUrl: "", }); - return cli.clearStores(); + + const clear = async() => { + await EventIndexPeg.deleteEventIndex(); + await cli.clearStores(); + } + + return clear(); } /** From 1cc64f2426bc049257985b06855b9ba9dbcd0113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 10:10:35 +0100 Subject: [PATCH 18/64] Searching: Move the small helper functions out of the eventIndexSearch function. --- src/Searching.js | 146 +++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index cd06d9bc67..cff5742b04 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -34,80 +34,80 @@ function serverSideSearch(term, roomId = undefined) { return searchPromise; } +async function combinedSearchFunc(searchTerm) { + // Create two promises, one for the local search, one for the + // server-side search. + const client = MatrixClientPeg.get(); + const serverSidePromise = serverSideSearch(searchTerm); + const localPromise = localSearchFunc(searchTerm); + + // Wait for both promises to resolve. + await Promise.all([serverSidePromise, localPromise]); + + // Get both search results. + const localResult = await localPromise; + const serverSideResult = await serverSidePromise; + + // Combine the search results into one result. + const result = {}; + + // Our localResult and serverSideResult are both ordered by + // recency separetly, when we combine them the order might not + // be the right one so we need to sort them. + const compare = (a, b) => { + const aEvent = a.context.getEvent().event; + const bEvent = b.context.getEvent().event; + + if (aEvent.origin_server_ts > + bEvent.origin_server_ts) return -1; + if (aEvent.origin_server_ts < + bEvent.origin_server_ts) return 1; + return 0; + }; + + result.count = localResult.count + serverSideResult.count; + result.results = localResult.results.concat( + serverSideResult.results).sort(compare); + result.highlights = localResult.highlights.concat( + serverSideResult.highlights); + + return result; +} + +async function localSearchFunc(searchTerm, roomId = undefined) { + const searchArgs = { + search_term: searchTerm, + before_limit: 1, + after_limit: 1, + order_by_recency: true, + }; + + if (roomId !== undefined) { + searchArgs.room_id = roomId; + } + + const eventIndex = EventIndexPeg.get(); + + const localResult = await eventIndex.search(searchArgs); + + const response = { + search_categories: { + room_events: localResult, + }, + }; + + const emptyResult = { + results: [], + highlights: [], + }; + + const result = MatrixClientPeg.get()._processRoomEventsSearch( + emptyResult, response); + + return result; +} + function eventIndexSearch(term, roomId = undefined) { - const combinedSearchFunc = async (searchTerm) => { - // Create two promises, one for the local search, one for the - // server-side search. - const client = MatrixClientPeg.get(); - const serverSidePromise = serverSideSearch(searchTerm); - const localPromise = localSearchFunc(searchTerm); - - // Wait for both promises to resolve. - await Promise.all([serverSidePromise, localPromise]); - - // Get both search results. - const localResult = await localPromise; - const serverSideResult = await serverSidePromise; - - // Combine the search results into one result. - const result = {}; - - // Our localResult and serverSideResult are both ordered by - // recency separetly, when we combine them the order might not - // be the right one so we need to sort them. - const compare = (a, b) => { - const aEvent = a.context.getEvent().event; - const bEvent = b.context.getEvent().event; - - if (aEvent.origin_server_ts > - bEvent.origin_server_ts) return -1; - if (aEvent.origin_server_ts < - bEvent.origin_server_ts) return 1; - return 0; - }; - - result.count = localResult.count + serverSideResult.count; - result.results = localResult.results.concat( - serverSideResult.results).sort(compare); - result.highlights = localResult.highlights.concat( - serverSideResult.highlights); - - return result; - }; - - const localSearchFunc = async (searchTerm, roomId = undefined) => { - const searchArgs = { - search_term: searchTerm, - before_limit: 1, - after_limit: 1, - order_by_recency: true, - }; - - if (roomId !== undefined) { - searchArgs.room_id = roomId; - } - - const eventIndex = EventIndexPeg.get(); - - const localResult = await eventIndex.search(searchArgs); - - const response = { - search_categories: { - room_events: localResult, - }, - }; - - const emptyResult = { - results: [], - highlights: [], - }; - - const result = MatrixClientPeg.get()._processRoomEventsSearch( - emptyResult, response); - - return result; - }; - let searchPromise; if (roomId !== undefined) { From 1df28c75262e113ea0111a6cc0dccb74a512e93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 10:30:38 +0100 Subject: [PATCH 19/64] Fix some lint errors. --- src/EventIndexPeg.js | 12 +++++++----- src/Lifecycle.js | 4 ++-- src/MatrixClientPeg.js | 2 -- src/Searching.js | 5 ++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 86fb889c7a..15d34ea230 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -32,7 +32,9 @@ class EventIndexPeg { } /** - * Returns the current Event index object for the application. Can be null + * Get the current event index. + * + * @Returns The EventIndex object for the application. Can be null * if the platform doesn't support event indexing. */ get() { @@ -47,25 +49,25 @@ class EventIndexPeg { const platform = PlatformPeg.get(); if (!platform.supportsEventIndexing()) return false; - let index = new EventIndex(); + const index = new EventIndex(); const userId = MatrixClientPeg.get().getUserId(); // TODO log errors here and return false if it errors out. await index.init(userId); this.index = index; - return true + return true; } stop() { if (this.index === null) return; - index.stop(); + this.index.stop(); this.index = null; } async deleteEventIndex() { if (this.index === null) return; - index.deleteEventIndex(); + this.index.deleteEventIndex(); } } diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 1e68bcc062..aa900c81a1 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -633,10 +633,10 @@ function _clearStorage() { baseUrl: "", }); - const clear = async() => { + const clear = async () => { await EventIndexPeg.deleteEventIndex(); await cli.clearStores(); - } + }; return clear(); } diff --git a/src/MatrixClientPeg.js b/src/MatrixClientPeg.js index 6c5b465bb0..bebb254afc 100644 --- a/src/MatrixClientPeg.js +++ b/src/MatrixClientPeg.js @@ -30,8 +30,6 @@ import {verificationMethods} from 'matrix-js-sdk/lib/crypto'; import MatrixClientBackedSettingsHandler from "./settings/handlers/MatrixClientBackedSettingsHandler"; import * as StorageManager from './utils/StorageManager'; import IdentityAuthClient from './IdentityAuthClient'; -import PlatformPeg from "./PlatformPeg"; -import EventIndexPeg from "./EventIndexPeg"; interface MatrixClientCreds { homeserverUrl: string, diff --git a/src/Searching.js b/src/Searching.js index cff5742b04..84e73b91f4 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -26,7 +26,7 @@ function serverSideSearch(term, roomId = undefined) { }; } - let searchPromise = MatrixClientPeg.get().searchRoomEvents({ + const searchPromise = MatrixClientPeg.get().searchRoomEvents({ filter: filter, term: term, }); @@ -37,7 +37,6 @@ function serverSideSearch(term, roomId = undefined) { async function combinedSearchFunc(searchTerm) { // Create two promises, one for the local search, one for the // server-side search. - const client = MatrixClientPeg.get(); const serverSidePromise = serverSideSearch(searchTerm); const localPromise = localSearchFunc(searchTerm); @@ -126,7 +125,7 @@ function eventIndexSearch(term, roomId = undefined) { searchPromise = combinedSearchFunc(term); } - return searchPromise + return searchPromise; } export default function eventSearch(term, roomId = undefined) { From 54b352f69cd1e9d82fff759c6838af1affca4f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 10:37:20 +0100 Subject: [PATCH 20/64] MatrixChat: Fix the limited timeline checkpoint adding. --- src/EventIndexing.js | 9 ++------- src/components/structures/MatrixChat.js | 4 +--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index ebd2ffe983..bf3f50690f 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -347,15 +347,10 @@ export default class EventIndexer { console.log("Seshat: Stopping crawler function"); } - async addCheckpointForLimitedRoom(roomId) { + async addCheckpointForLimitedRoom(room) { const platform = PlatformPeg.get(); if (!platform.supportsEventIndexing()) return; - if (!MatrixClientPeg.get().isRoomEncrypted(roomId)) return; - - const client = MatrixClientPeg.get(); - const room = client.getRoom(roomId); - - if (room === null) return; + if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; const timeline = room.getLiveTimeline(); const token = timeline.getPaginationToken("b"); diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index 0d3d5abd55..ccc8b5e1d6 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1223,8 +1223,6 @@ export default createReactClass({ * Called when the session is logged out */ _onLoggedOut: async function() { - const platform = PlatformPeg.get(); - this.notifyNewScreen('login'); this.setStateForNewView({ view: VIEWS.LOGIN, @@ -1313,7 +1311,7 @@ export default createReactClass({ const eventIndex = EventIndexPeg.get(); if (eventIndex === null) return; if (resetAllTimelines === true) return; - await eventIndex.addCheckpointForLimitedRoom(roomId); + await eventIndex.addCheckpointForLimitedRoom(room); }); cli.on('sync', function(state, prevState, data) { From 80b28004e15821bd127bee3121baabd1cf6226a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 11:02:54 +0100 Subject: [PATCH 21/64] Searching: Define the room id in the const object. --- src/Searching.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Searching.js b/src/Searching.js index 84e73b91f4..ee46a66fb8 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -79,6 +79,7 @@ async function localSearchFunc(searchTerm, roomId = undefined) { before_limit: 1, after_limit: 1, order_by_recency: true, + room_id: undefined, }; if (roomId !== undefined) { From f453fea24acf110d0b297d8374234a8c873bec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 12:25:16 +0100 Subject: [PATCH 22/64] BasePlatform: Move the event indexing methods into a separate class. --- src/BaseEventIndexManager.js | 208 +++++++++++++++++++++++++++++++++++ src/BasePlatform.js | 41 +------ src/EventIndexPeg.js | 6 +- src/EventIndexing.js | 62 +++++------ 4 files changed, 246 insertions(+), 71 deletions(-) create mode 100644 src/BaseEventIndexManager.js diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js new file mode 100644 index 0000000000..cd7a735e8d --- /dev/null +++ b/src/BaseEventIndexManager.js @@ -0,0 +1,208 @@ +// @flow + +/* +Copyright 2019 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export interface MatrixEvent { + type: string; + sender: string; + content: {}; + event_id: string; + origin_server_ts: number; + unsigned: ?{}; + room_id: string; +} + +export interface MatrixProfile { + avatar_url: string; + displayname: string; +} + +export interface CrawlerCheckpoint { + roomId: string; + token: string; + fullCrawl: boolean; + direction: string; +} + +export interface ResultContext { + events_before: [MatrixEvent]; + events_after: [MatrixEvent]; + profile_info: Map; +} + +export interface ResultsElement { + rank: number; + result: MatrixEvent; + context: ResultContext; +} + +export interface SearchResult { + count: number; + results: [ResultsElement]; + highlights: [string]; +} + +export interface SearchArgs { + search_term: string; + before_limit: number; + after_limit: number; + order_by_recency: boolean; + room_id: ?string; +} + +export interface HistoricEvent { + event: MatrixEvent; + profile: MatrixProfile; +} + +/** + * Base class for classes that provide platform-specific event indexing. + * + * Instances of this class are provided by the application. + */ +export default class BaseEventIndexManager { + /** + * Initialize the event index for the given user. + * + * @param {string} userId The unique identifier of the logged in user that + * owns the index. + * + * @return {Promise} A promise that will resolve when the event index is + * initialized. + */ + async initEventIndex(userId: string): Promise<> { + throw new Error("Unimplemented"); + } + + /** + * Queue up an event to be added to the index. + * + * @param {MatrixEvent} ev The event that should be added to the index. + * @param {MatrixProfile} profile The profile of the event sender at the + * time of the event receival. + * + * @return {Promise} A promise that will resolve when the was queued up for + * addition. + */ + async addEventToIndex(ev: MatrixEvent, profile: MatrixProfile): Promise<> { + throw new Error("Unimplemented"); + } + + /** + * Check if our event index is empty. + * + * @return {Promise} A promise that will resolve to true if the + * event index is empty, false otherwise. + */ + indexIsEmpty(): Promise { + throw new Error("Unimplemented"); + } + + /** + * Commit the previously queued up events to the index. + * + * @return {Promise} A promise that will resolve once the queued up events + * were added to the index. + */ + async commitLiveEvents(): Promise<> { + throw new Error("Unimplemented"); + } + + /** + * Search the event index using the given term for matching events. + * + * @param {SearchArgs} searchArgs The search configuration sets what should + * be searched for and what should be contained in the search result. + * + * @return {Promise<[SearchResult]>} A promise that will resolve to an array + * of search results once the search is done. + */ + async searchEventIndex(searchArgs: SearchArgs): Promise { + throw new Error("Unimplemented"); + } + + /** + * Add events from the room history to the event index. + * + * This is used to add a batch of events to the index. + * + * @param {[HistoricEvent]} events The list of events and profiles that + * should be added to the event index. + * @param {[CrawlerCheckpoint]} checkpoint A new crawler checkpoint that + * should be stored in the index which should be used to continue crawling + * the room. + * @param {[CrawlerCheckpoint]} oldCheckpoint The checkpoint that was used + * to fetch the current batch of events. This checkpoint will be removed + * from the index. + * + * @return {Promise} A promise that will resolve to true if all the events + * were already added to the index, false otherwise. + */ + async addHistoricEvents( + events: [HistoricEvent], + checkpoint: CrawlerCheckpoint | null = null, + oldCheckpoint: CrawlerCheckpoint | null = null, + ): Promise { + throw new Error("Unimplemented"); + } + + /** + * Add a new crawler checkpoint to the index. + * + * @param {CrawlerCheckpoint} checkpoint The checkpoint that should be added + * to the index. + * + * @return {Promise} A promise that will resolve once the checkpoint has + * been stored. + */ + async addCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + throw new Error("Unimplemented"); + } + + /** + * Add a new crawler checkpoint to the index. + * + * @param {CrawlerCheckpoint} checkpoint The checkpoint that should be + * removed from the index. + * + * @return {Promise} A promise that will resolve once the checkpoint has + * been removed. + */ + async removeCrawlerCheckpoint(checkpoint: CrawlerCheckpoint): Promise<> { + throw new Error("Unimplemented"); + } + + /** + * Load the stored checkpoints from the index. + * + * @return {Promise<[CrawlerCheckpoint]>} A promise that will resolve to an + * array of crawler checkpoints once they have been loaded from the index. + */ + async loadCheckpoints(): Promise<[CrawlerCheckpoint]> { + throw new Error("Unimplemented"); + } + + /** + * Delete our current event index. + * + * @return {Promise} A promise that will resolve once the event index has + * been deleted. + */ + async deleteEventIndex(): Promise<> { + throw new Error("Unimplemented"); + } +} diff --git a/src/BasePlatform.js b/src/BasePlatform.js index 7f5df822e4..582ac24cb0 100644 --- a/src/BasePlatform.js +++ b/src/BasePlatform.js @@ -19,6 +19,7 @@ limitations under the License. */ import dis from './dispatcher'; +import BaseEventIndexManager from './BaseEventIndexManager'; /** * Base class for classes that provide platform-specific functionality @@ -152,43 +153,7 @@ export default class BasePlatform { throw new Error("Unimplemented"); } - supportsEventIndexing(): boolean { - return false; - } - - async initEventIndex(userId: string): boolean { - throw new Error("Unimplemented"); - } - - async addEventToIndex(ev: {}, profile: {}): void { - throw new Error("Unimplemented"); - } - - indexIsEmpty(): Promise { - throw new Error("Unimplemented"); - } - - async commitLiveEvents(): void { - throw new Error("Unimplemented"); - } - - async searchEventIndex(term: string): Promise<{}> { - throw new Error("Unimplemented"); - } - - async addHistoricEvents(events: [], checkpoint: {} = null, oldCheckpoint: {} = null): Promise { - throw new Error("Unimplemented"); - } - - async addCrawlerCheckpoint(checkpoint: {}): Promise<> { - throw new Error("Unimplemented"); - } - - async removeCrawlerCheckpoint(checkpoint: {}): Promise<> { - throw new Error("Unimplemented"); - } - - async deleteEventIndex(): Promise<> { - throw new Error("Unimplemented"); + getEventIndexingManager(): BaseEventIndexManager | null { + return null; } } diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 15d34ea230..bec3f075b6 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -46,9 +46,11 @@ class EventIndexPeg { * otherwise. */ async init() { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return false; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + console.log("Initializing event index, got {}", indexManager); + if (indexManager === null) return false; + console.log("Seshat: Creatingnew EventIndex object", indexManager); const index = new EventIndex(); const userId = MatrixClientPeg.get().getUserId(); diff --git a/src/EventIndexing.js b/src/EventIndexing.js index bf3f50690f..60482b76b5 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -31,19 +31,19 @@ export default class EventIndexer { } async init(userId) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return false; - platform.initEventIndex(userId); + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return false; + indexManager.initEventIndex(userId); return true; } async onSync(state, prevState, data) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return; if (prevState === null && state === "PREPARED") { // Load our stored checkpoints, if any. - this.crawlerChekpoints = await platform.loadCheckpoints(); + this.crawlerChekpoints = await indexManager.loadCheckpoints(); console.log("Seshat: Loaded checkpoints", this.crawlerChekpoints); return; @@ -85,8 +85,8 @@ export default class EventIndexer { direction: "f", }; - await platform.addCrawlerCheckpoint(backCheckpoint); - await platform.addCrawlerCheckpoint(forwardCheckpoint); + await indexManager.addCrawlerCheckpoint(backCheckpoint); + await indexManager.addCrawlerCheckpoint(forwardCheckpoint); this.crawlerChekpoints.push(backCheckpoint); this.crawlerChekpoints.push(forwardCheckpoint); })); @@ -95,7 +95,7 @@ export default class EventIndexer { // If our indexer is empty we're most likely running Riot the // first time with indexing support or running it with an // initial sync. Add checkpoints to crawl our encrypted rooms. - const eventIndexWasEmpty = await platform.isEventIndexEmpty(); + const eventIndexWasEmpty = await indexManager.isEventIndexEmpty(); if (eventIndexWasEmpty) await addInitialCheckpoints(); // Start our crawler. @@ -107,14 +107,14 @@ export default class EventIndexer { // A sync was done, presumably we queued up some live events, // commit them now. console.log("Seshat: Committing events"); - await platform.commitLiveEvents(); + await indexManager.commitLiveEvents(); return; } } async onRoomTimeline(ev, room, toStartOfTimeline, removed, data) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return; // We only index encrypted rooms locally. if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -139,8 +139,8 @@ export default class EventIndexer { } async onEventDecrypted(ev, err) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return; const eventId = ev.getId(); @@ -151,8 +151,8 @@ export default class EventIndexer { } async addLiveEventToIndex(ev) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return; if (["m.room.message", "m.room.name", "m.room.topic"] .indexOf(ev.getType()) == -1) { @@ -165,7 +165,7 @@ export default class EventIndexer { avatar_url: ev.sender.getMxcAvatarUrl(), }; - platform.addEventToIndex(e, profile); + indexManager.addEventToIndex(e, profile); } async crawlerFunc(handle) { @@ -180,7 +180,7 @@ export default class EventIndexer { console.log("Seshat: Started crawler function"); const client = MatrixClientPeg.get(); - const platform = PlatformPeg.get(); + const indexManager = PlatformPeg.get().getEventIndexingManager(); handle.cancel = () => { cancelled = true; @@ -223,14 +223,14 @@ export default class EventIndexer { } catch (e) { console.log("Seshat: Error crawling events:", e); this.crawlerChekpoints.push(checkpoint); - continue + continue; } if (res.chunk.length === 0) { console.log("Seshat: Done with the checkpoint", checkpoint); // We got to the start/end of our timeline, lets just // delete our checkpoint and go back to sleep. - await platform.removeCrawlerCheckpoint(checkpoint); + await indexManager.removeCrawlerCheckpoint(checkpoint); continue; } @@ -323,7 +323,7 @@ export default class EventIndexer { ); try { - const eventsAlreadyAdded = await platform.addHistoricEvents( + const eventsAlreadyAdded = await indexManager.addHistoricEvents( events, newCheckpoint, checkpoint); // If all events were already indexed we assume that we catched // up with our index and don't need to crawl the room further. @@ -332,7 +332,7 @@ export default class EventIndexer { if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { console.log("Seshat: Checkpoint had already all events", "added, stopping the crawl", checkpoint); - await platform.removeCrawlerCheckpoint(newCheckpoint); + await indexManager.removeCrawlerCheckpoint(newCheckpoint); } else { this.crawlerChekpoints.push(newCheckpoint); } @@ -348,8 +348,8 @@ export default class EventIndexer { } async addCheckpointForLimitedRoom(room) { - const platform = PlatformPeg.get(); - if (!platform.supportsEventIndexing()) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager === null) return; if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; const timeline = room.getLiveTimeline(); @@ -372,19 +372,19 @@ export default class EventIndexer { console.log("Seshat: Added checkpoint because of a limited timeline", backwardsCheckpoint, forwardsCheckpoint); - await platform.addCrawlerCheckpoint(backwardsCheckpoint); - await platform.addCrawlerCheckpoint(forwardsCheckpoint); + await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); + await indexManager.addCrawlerCheckpoint(forwardsCheckpoint); this.crawlerChekpoints.push(backwardsCheckpoint); this.crawlerChekpoints.push(forwardsCheckpoint); } async deleteEventIndex() { - const platform = PlatformPeg.get(); - if (platform.supportsEventIndexing()) { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + if (indexManager !== null) { console.log("Seshat: Deleting event index."); this.crawlerRef.cancel(); - await platform.deleteEventIndex(); + await indexManager.deleteEventIndex(); } } @@ -400,7 +400,7 @@ export default class EventIndexer { } async search(searchArgs) { - const platform = PlatformPeg.get(); - return platform.searchEventIndex(searchArgs) + const indexManager = PlatformPeg.get().getEventIndexingManager(); + return indexManager.searchEventIndex(searchArgs); } } From 1316e04776b90ec7cc3d7770822b400795de171b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 15:23:08 +0100 Subject: [PATCH 23/64] EventIndexing: Check if there is a room when resetting the timeline. --- src/EventIndexing.js | 13 ++----------- src/components/structures/MatrixChat.js | 4 ++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 60482b76b5..4817df4b32 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -347,7 +347,7 @@ export default class EventIndexer { console.log("Seshat: Stopping crawler function"); } - async addCheckpointForLimitedRoom(room) { + async onLimitedTimeline(room) { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (indexManager === null) return; if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -362,21 +362,12 @@ export default class EventIndexer { direction: "b", }; - const forwardsCheckpoint = { - roomId: room.roomId, - token: token, - fullCrawl: false, - direction: "f", - }; - console.log("Seshat: Added checkpoint because of a limited timeline", - backwardsCheckpoint, forwardsCheckpoint); + backwardsCheckpoint); await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); - await indexManager.addCrawlerCheckpoint(forwardsCheckpoint); this.crawlerChekpoints.push(backwardsCheckpoint); - this.crawlerChekpoints.push(forwardsCheckpoint); } async deleteEventIndex() { diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index ccc8b5e1d6..f78bb5c168 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1310,8 +1310,8 @@ export default createReactClass({ cli.on("Room.timelineReset", async (room, timelineSet, resetAllTimelines) => { const eventIndex = EventIndexPeg.get(); if (eventIndex === null) return; - if (resetAllTimelines === true) return; - await eventIndex.addCheckpointForLimitedRoom(room); + if (room === null) return; + await eventIndex.onLimitedTimeline(room); }); cli.on('sync', function(state, prevState, data) { From ab7f34b45a66748fde1ee361faa7f31bc86db0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 15:26:27 +0100 Subject: [PATCH 24/64] EventIndexing: Don't mention Seshat in the logs. --- src/EventIndexing.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 4817df4b32..f67d4c9eb3 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -44,7 +44,7 @@ export default class EventIndexer { if (prevState === null && state === "PREPARED") { // Load our stored checkpoints, if any. this.crawlerChekpoints = await indexManager.loadCheckpoints(); - console.log("Seshat: Loaded checkpoints", + console.log("EventIndex: Loaded checkpoints", this.crawlerChekpoints); return; } @@ -62,7 +62,7 @@ export default class EventIndexer { // rooms can use the search provided by the Homeserver. const encryptedRooms = rooms.filter(isRoomEncrypted); - console.log("Seshat: Adding initial crawler checkpoints"); + console.log("EventIndex: Adding initial crawler checkpoints"); // Gather the prev_batch tokens and create checkpoints for // our message crawler. @@ -70,7 +70,7 @@ export default class EventIndexer { const timeline = room.getLiveTimeline(); const token = timeline.getPaginationToken("b"); - console.log("Seshat: Got token for indexer", + console.log("EventIndex: Got token for indexer", room.roomId, token); const backCheckpoint = { @@ -106,7 +106,7 @@ export default class EventIndexer { if (prevState === "SYNCING" && state === "SYNCING") { // A sync was done, presumably we queued up some live events, // commit them now. - console.log("Seshat: Committing events"); + console.log("EventIndex: Committing events"); await indexManager.commitLiveEvents(); return; } @@ -177,7 +177,7 @@ export default class EventIndexer { let cancelled = false; - console.log("Seshat: Started crawler function"); + console.log("EventIndex: Started crawler function"); const client = MatrixClientPeg.get(); const indexManager = PlatformPeg.get().getEventIndexingManager(); @@ -192,10 +192,10 @@ export default class EventIndexer { // here. await sleep(this._crawler_timeout); - console.log("Seshat: Running the crawler loop."); + console.log("EventIndex: Running the crawler loop."); if (cancelled) { - console.log("Seshat: Cancelling the crawler."); + console.log("EventIndex: Cancelling the crawler."); break; } @@ -207,7 +207,7 @@ export default class EventIndexer { continue; } - console.log("Seshat: crawling using checkpoint", checkpoint); + console.log("EventIndex: crawling using checkpoint", checkpoint); // We have a checkpoint, let us fetch some messages, again, very // conservatively to not bother our Homeserver too much. @@ -221,13 +221,13 @@ export default class EventIndexer { checkpoint.roomId, checkpoint.token, 100, checkpoint.direction); } catch (e) { - console.log("Seshat: Error crawling events:", e); + console.log("EventIndex: Error crawling events:", e); this.crawlerChekpoints.push(checkpoint); continue; } if (res.chunk.length === 0) { - console.log("Seshat: Done with the checkpoint", checkpoint); + console.log("EventIndex: Done with the checkpoint", checkpoint); // We got to the start/end of our timeline, lets just // delete our checkpoint and go back to sleep. await indexManager.removeCrawlerCheckpoint(checkpoint); @@ -289,7 +289,7 @@ export default class EventIndexer { // stage? const filteredEvents = matrixEvents.filter(isValidEvent); - // Let us convert the events back into a format that Seshat can + // Let us convert the events back into a format that EventIndex can // consume. const events = filteredEvents.map((ev) => { const jsonEvent = ev.toJSON(); @@ -317,7 +317,7 @@ export default class EventIndexer { }; console.log( - "Seshat: Crawled room", + "EventIndex: Crawled room", client.getRoom(checkpoint.roomId).name, "and fetched", events.length, "events.", ); @@ -330,21 +330,21 @@ export default class EventIndexer { // Let us delete the checkpoint in that case, otherwise push // the new checkpoint to be used by the crawler. if (eventsAlreadyAdded === true && newCheckpoint.fullCrawl !== true) { - console.log("Seshat: Checkpoint had already all events", + console.log("EventIndex: Checkpoint had already all events", "added, stopping the crawl", checkpoint); await indexManager.removeCrawlerCheckpoint(newCheckpoint); } else { this.crawlerChekpoints.push(newCheckpoint); } } catch (e) { - console.log("Seshat: Error durring a crawl", e); + console.log("EventIndex: Error durring a crawl", e); // An error occured, put the checkpoint back so we // can retry. this.crawlerChekpoints.push(checkpoint); } } - console.log("Seshat: Stopping crawler function"); + console.log("EventIndex: Stopping crawler function"); } async onLimitedTimeline(room) { @@ -362,7 +362,7 @@ export default class EventIndexer { direction: "b", }; - console.log("Seshat: Added checkpoint because of a limited timeline", + console.log("EventIndex: Added checkpoint because of a limited timeline", backwardsCheckpoint); await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); @@ -373,7 +373,7 @@ export default class EventIndexer { async deleteEventIndex() { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (indexManager !== null) { - console.log("Seshat: Deleting event index."); + console.log("EventIndex: Deleting event index."); this.crawlerRef.cancel(); await indexManager.deleteEventIndex(); } From c33f5ba0ca8292116e1623a9d0c932aac62479a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 15:39:06 +0100 Subject: [PATCH 25/64] BaseEventIndexManager: Add a method to perform runtime checks for indexing support. --- src/BaseEventIndexManager.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index cd7a735e8d..a74eac658a 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -75,6 +75,19 @@ export interface HistoricEvent { * Instances of this class are provided by the application. */ export default class BaseEventIndexManager { + /** + * Does our EventIndexManager support event indexing. + * + * If an EventIndexManager imlpementor has runtime dependencies that + * optionally enable event indexing they may override this method to perform + * the necessary runtime checks here. + * + * @return {Promise} A promise that will resolve to true if event indexing + * is supported, false otherwise. + */ + async supportsEventIndexing(): Promise { + return true; + } /** * Initialize the event index for the given user. * From bf558b46c3cfc9ee7b19dbe7a92ac79ed118e498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 15:39:39 +0100 Subject: [PATCH 26/64] EventIndexPeg: Clean up the event index initialization. --- src/EventIndexPeg.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index bec3f075b6..3ce88339eb 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -47,15 +47,25 @@ class EventIndexPeg { */ async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); - console.log("Initializing event index, got {}", indexManager); if (indexManager === null) return false; - console.log("Seshat: Creatingnew EventIndex object", indexManager); - const index = new EventIndex(); + if (await indexManager.supportsEventIndexing() !== true) { + console.log("EventIndex: Platform doesn't support event indexing,", + "not initializing."); + return false; + } + const index = new EventIndex(); const userId = MatrixClientPeg.get().getUserId(); - // TODO log errors here and return false if it errors out. - await index.init(userId); + + try { + await index.init(userId); + } catch (e) { + console.log("EventIndex: Error initializing the event index", e); + } + + console.log("EventIndex: Successfully initialized the event index"); + this.index = index; return true; From c26df9d9efc836b1a6b5d660edd702448a22b3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 15:57:12 +0100 Subject: [PATCH 27/64] EventIndexing: Fix a typo. --- src/EventIndexing.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index f67d4c9eb3..af77979040 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -22,7 +22,7 @@ import MatrixClientPeg from "./MatrixClientPeg"; */ export default class EventIndexer { constructor() { - this.crawlerChekpoints = []; + this.crawlerCheckpoints = []; // The time that the crawler will wait between /rooms/{room_id}/messages // requests this._crawler_timeout = 3000; @@ -43,9 +43,9 @@ export default class EventIndexer { if (prevState === null && state === "PREPARED") { // Load our stored checkpoints, if any. - this.crawlerChekpoints = await indexManager.loadCheckpoints(); + this.crawlerCheckpoints = await indexManager.loadCheckpoints(); console.log("EventIndex: Loaded checkpoints", - this.crawlerChekpoints); + this.crawlerCheckpoints); return; } @@ -87,8 +87,8 @@ export default class EventIndexer { await indexManager.addCrawlerCheckpoint(backCheckpoint); await indexManager.addCrawlerCheckpoint(forwardCheckpoint); - this.crawlerChekpoints.push(backCheckpoint); - this.crawlerChekpoints.push(forwardCheckpoint); + this.crawlerCheckpoints.push(backCheckpoint); + this.crawlerCheckpoints.push(forwardCheckpoint); })); }; @@ -199,7 +199,7 @@ export default class EventIndexer { break; } - const checkpoint = this.crawlerChekpoints.shift(); + const checkpoint = this.crawlerCheckpoints.shift(); /// There is no checkpoint available currently, one may appear if // a sync with limited room timelines happens, so go back to sleep. @@ -222,7 +222,7 @@ export default class EventIndexer { checkpoint.direction); } catch (e) { console.log("EventIndex: Error crawling events:", e); - this.crawlerChekpoints.push(checkpoint); + this.crawlerCheckpoints.push(checkpoint); continue; } @@ -334,13 +334,13 @@ export default class EventIndexer { "added, stopping the crawl", checkpoint); await indexManager.removeCrawlerCheckpoint(newCheckpoint); } else { - this.crawlerChekpoints.push(newCheckpoint); + this.crawlerCheckpoints.push(newCheckpoint); } } catch (e) { console.log("EventIndex: Error durring a crawl", e); // An error occured, put the checkpoint back so we // can retry. - this.crawlerChekpoints.push(checkpoint); + this.crawlerCheckpoints.push(checkpoint); } } @@ -367,7 +367,7 @@ export default class EventIndexer { await indexManager.addCrawlerCheckpoint(backwardsCheckpoint); - this.crawlerChekpoints.push(backwardsCheckpoint); + this.crawlerCheckpoints.push(backwardsCheckpoint); } async deleteEventIndex() { From cc2ee53824b955e513def52cf4a08118d853e646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 16:21:26 +0100 Subject: [PATCH 28/64] EventIndex: Add some more docs and fix some lint issues. --- src/BaseEventIndexManager.js | 2 +- src/BasePlatform.js | 6 ++++++ src/EventIndexPeg.js | 20 ++++++++++++++++---- src/components/structures/RoomView.js | 2 +- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index a74eac658a..48a96c4d88 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -168,7 +168,7 @@ export default class BaseEventIndexManager { async addHistoricEvents( events: [HistoricEvent], checkpoint: CrawlerCheckpoint | null = null, - oldCheckpoint: CrawlerCheckpoint | null = null, + oldCheckpoint: CrawlerCheckpoint | null = null, ): Promise { throw new Error("Unimplemented"); } diff --git a/src/BasePlatform.js b/src/BasePlatform.js index 582ac24cb0..f6301fd173 100644 --- a/src/BasePlatform.js +++ b/src/BasePlatform.js @@ -153,6 +153,12 @@ export default class BasePlatform { throw new Error("Unimplemented"); } + /** + * Get our platform specific EventIndexManager. + * + * @return {BaseEventIndexManager} The EventIndex manager for our platform, + * can be null if the platform doesn't support event indexing. + */ getEventIndexingManager(): BaseEventIndexManager | null { return null; } diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 3ce88339eb..1b380e273f 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -34,16 +34,16 @@ class EventIndexPeg { /** * Get the current event index. * - * @Returns The EventIndex object for the application. Can be null - * if the platform doesn't support event indexing. + * @return {EventIndex} The current event index. */ get() { return this.index; } /** Create a new EventIndex and initialize it if the platform supports it. - * Returns true if an EventIndex was successfully initialized, false - * otherwise. + * + * @return {Promise} A promise that will resolve to true if an + * EventIndex was successfully initialized, false otherwise. */ async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); @@ -71,15 +71,27 @@ class EventIndexPeg { return true; } + /** + * Stop our event indexer. + */ stop() { if (this.index === null) return; this.index.stop(); this.index = null; } + /** + * Delete our event indexer. + * + * After a call to this the init() method will need to be called again. + * + * @return {Promise} A promise that will resolve once the event index is + * deleted. + */ async deleteEventIndex() { if (this.index === null) return; this.index.deleteEventIndex(); + this.index = null; } } diff --git a/src/components/structures/RoomView.js b/src/components/structures/RoomView.js index 9fe54ad164..6dee60bec7 100644 --- a/src/components/structures/RoomView.js +++ b/src/components/structures/RoomView.js @@ -1131,7 +1131,7 @@ module.exports = createReactClass({ this.searchId = new Date().getTime(); let roomId; - if (scope === "Room") roomId = this.state.room.roomId, + if (scope === "Room") roomId = this.state.room.roomId; debuglog("sending search request"); const searchPromise = eventSearch(term, roomId); From 368a77ec3ef318f7e0e55832bf97877b8575f737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 16:35:04 +0100 Subject: [PATCH 29/64] EventIndexing: Fix a style issue. --- src/EventIndexing.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index af77979040..5830106e84 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -25,7 +25,7 @@ export default class EventIndexer { this.crawlerCheckpoints = []; // The time that the crawler will wait between /rooms/{room_id}/messages // requests - this._crawler_timeout = 3000; + this._crawlerTimeout = 3000; this._crawlerRef = null; this.liveEventsForIndex = new Set(); } From d4b31cb7e037301b0786372d3ae643c96b2b48e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 16:35:26 +0100 Subject: [PATCH 30/64] EventIndexing: Move the max events per crawl constant into the class. --- src/EventIndexing.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 5830106e84..77c4022480 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -26,6 +26,9 @@ export default class EventIndexer { // The time that the crawler will wait between /rooms/{room_id}/messages // requests this._crawlerTimeout = 3000; + // The maximum number of events our crawler should fetch in a single + // crawl. + this._eventsPerCrawl = 100; this._crawlerRef = null; this.liveEventsForIndex = new Set(); } @@ -218,7 +221,7 @@ export default class EventIndexer { try { res = await client._createMessagesRequest( - checkpoint.roomId, checkpoint.token, 100, + checkpoint.roomId, checkpoint.token, this._eventsPerCrawl, checkpoint.direction); } catch (e) { console.log("EventIndex: Error crawling events:", e); From 9b32ec10b43cc274df28d938610fbf8c4b53479b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 13 Nov 2019 16:47:21 +0100 Subject: [PATCH 31/64] EventIndexing: Use the correct timeout value. --- src/EventIndexing.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 77c4022480..67bd894c67 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -193,7 +193,7 @@ export default class EventIndexer { // This is a low priority task and we don't want to spam our // Homeserver with /messages requests so we set a hefty timeout // here. - await sleep(this._crawler_timeout); + await sleep(this._crawlerTimeout); console.log("EventIndex: Running the crawler loop."); From 28d2e658a4d184d7f51d2423cc0cde5c6ad41986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 14 Nov 2019 14:13:49 +0100 Subject: [PATCH 32/64] EventIndexing: Don't scope the event index per user. --- src/BaseEventIndexManager.js | 5 +---- src/EventIndexPeg.js | 3 +-- src/EventIndexing.js | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index 48a96c4d88..073bdbec81 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -91,13 +91,10 @@ export default class BaseEventIndexManager { /** * Initialize the event index for the given user. * - * @param {string} userId The unique identifier of the logged in user that - * owns the index. - * * @return {Promise} A promise that will resolve when the event index is * initialized. */ - async initEventIndex(userId: string): Promise<> { + async initEventIndex(): Promise<> { throw new Error("Unimplemented"); } diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 1b380e273f..ff1b2099f2 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -56,10 +56,9 @@ class EventIndexPeg { } const index = new EventIndex(); - const userId = MatrixClientPeg.get().getUserId(); try { - await index.init(userId); + await index.init(); } catch (e) { console.log("EventIndex: Error initializing the event index", e); } diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 67bd894c67..1fc9197082 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -33,10 +33,10 @@ export default class EventIndexer { this.liveEventsForIndex = new Set(); } - async init(userId) { + async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (indexManager === null) return false; - indexManager.initEventIndex(userId); + indexManager.initEventIndex(); return true; } From 448c9a82908b9e1504ed28a66dd1a68cb9daf9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 14 Nov 2019 16:01:14 +0100 Subject: [PATCH 33/64] EventIndexPeg: Add a missing return statement. --- src/EventIndexPeg.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index ff1b2099f2..a4ab1815c9 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -61,6 +61,7 @@ class EventIndexPeg { await index.init(); } catch (e) { console.log("EventIndex: Error initializing the event index", e); + return false; } console.log("EventIndex: Successfully initialized the event index"); From 7516f2724aeb34f13ae379f7d5c2124beca1b5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 14 Nov 2019 16:13:22 +0100 Subject: [PATCH 34/64] EventIndexing: Rework the index initialization and deletion. --- src/BaseEventIndexManager.js | 10 +++++++++ src/EventIndexPeg.js | 43 ++++++++++++++++++++++++------------ src/EventIndexing.js | 40 +++++++++++++++++++-------------- src/Lifecycle.js | 1 + 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index 073bdbec81..4e52344e76 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -206,6 +206,16 @@ export default class BaseEventIndexManager { throw new Error("Unimplemented"); } + /** + * close our event index. + * + * @return {Promise} A promise that will resolve once the event index has + * been closed. + */ + async closeEventIndex(): Promise<> { + throw new Error("Unimplemented"); + } + /** * Delete our current event index. * diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index a4ab1815c9..dc25b11cf7 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -31,15 +31,6 @@ class EventIndexPeg { this.index = null; } - /** - * Get the current event index. - * - * @return {EventIndex} The current event index. - */ - get() { - return this.index; - } - /** Create a new EventIndex and initialize it if the platform supports it. * * @return {Promise} A promise that will resolve to true if an @@ -72,11 +63,30 @@ class EventIndexPeg { } /** - * Stop our event indexer. + * Get the current event index. + * + * @return {EventIndex} The current event index. */ + get() { + return this.index; + } + stop() { if (this.index === null) return; - this.index.stop(); + this.index.stopCrawler(); + } + + /** + * Unset our event store + * + * After a call to this the init() method will need to be called again. + * + * @return {Promise} A promise that will resolve once the event index is + * closed. + */ + async unset() { + if (this.index === null) return; + this.index.close(); this.index = null; } @@ -89,9 +99,14 @@ class EventIndexPeg { * deleted. */ async deleteEventIndex() { - if (this.index === null) return; - this.index.deleteEventIndex(); - this.index = null; + const indexManager = PlatformPeg.get().getEventIndexingManager(); + + if (indexManager !== null) { + this.stop(); + console.log("EventIndex: Deleting event index."); + await indexManager.deleteEventIndex(); + this.index = null; + } } } diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 1fc9197082..37167cf600 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -35,9 +35,7 @@ export default class EventIndexer { async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return false; - indexManager.initEventIndex(); - return true; + return indexManager.initEventIndex(); } async onSync(state, prevState, data) { @@ -198,7 +196,6 @@ export default class EventIndexer { console.log("EventIndex: Running the crawler loop."); if (cancelled) { - console.log("EventIndex: Cancelling the crawler."); break; } @@ -373,26 +370,35 @@ export default class EventIndexer { this.crawlerCheckpoints.push(backwardsCheckpoint); } + startCrawler() { + if (this._crawlerRef !== null) return; + + const crawlerHandle = {}; + this.crawlerFunc(crawlerHandle); + this._crawlerRef = crawlerHandle; + } + + stopCrawler() { + if (this._crawlerRef === null) return; + + this._crawlerRef.cancel(); + this._crawlerRef = null; + } + + async close() { + const indexManager = PlatformPeg.get().getEventIndexingManager(); + this.stopCrawler(); + return indexManager.closeEventIndex(); + } + async deleteEventIndex() { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (indexManager !== null) { - console.log("EventIndex: Deleting event index."); - this.crawlerRef.cancel(); + this.stopCrawler(); await indexManager.deleteEventIndex(); } } - startCrawler() { - const crawlerHandle = {}; - this.crawlerFunc(crawlerHandle); - this.crawlerRef = crawlerHandle; - } - - stop() { - this._crawlerRef.cancel(); - this._crawlerRef = null; - } - async search(searchArgs) { const indexManager = PlatformPeg.get().getEventIndexingManager(); return indexManager.searchEventIndex(searchArgs); diff --git a/src/Lifecycle.js b/src/Lifecycle.js index aa900c81a1..1d38934ade 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -662,6 +662,7 @@ export function stopMatrixClient(unsetClient=true) { if (unsetClient) { MatrixClientPeg.unset(); + EventIndexPeg.unset().done(); } } } From d82d4246e92800588c77ed74f3e4f957a554ffbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 14 Nov 2019 16:17:50 +0100 Subject: [PATCH 35/64] BaseEventIndexManager: Remove a return from a docstring. --- src/BaseEventIndexManager.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index 4e52344e76..fe59cee673 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -114,9 +114,6 @@ export default class BaseEventIndexManager { /** * Check if our event index is empty. - * - * @return {Promise} A promise that will resolve to true if the - * event index is empty, false otherwise. */ indexIsEmpty(): Promise { throw new Error("Unimplemented"); From eb0b0a400f72d8ada1e9018192eff00c42dcf250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Thu, 14 Nov 2019 16:18:36 +0100 Subject: [PATCH 36/64] EventIndexPeg: Remove the now unused import of MatrixClientPeg. --- src/EventIndexPeg.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index dc25b11cf7..da5c5425e4 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -24,7 +24,6 @@ limitations under the License. import PlatformPeg from "./PlatformPeg"; import EventIndex from "./EventIndexing"; -import MatrixClientPeg from "./MatrixClientPeg"; class EventIndexPeg { constructor() { From 30d4dd36a7d2086123b52d95123cbd3e43122754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:05:11 +0100 Subject: [PATCH 37/64] BaseEventIndexManager: Remove the flow annotation. --- src/BaseEventIndexManager.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index fe59cee673..c5a3273a45 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -1,5 +1,3 @@ -// @flow - /* Copyright 2019 New Vector Ltd From ab93745460501c13ec9f68fe118fbaa9f2c06480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:16:29 +0100 Subject: [PATCH 38/64] Fix the copyright headers from New Vector to The Matrix Foundation. --- src/BaseEventIndexManager.js | 2 +- src/EventIndexPeg.js | 2 +- src/EventIndexing.js | 2 +- src/Searching.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index c5a3273a45..7cefb023d1 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -1,5 +1,5 @@ /* -Copyright 2019 New Vector Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index da5c5425e4..54d9c40079 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -1,5 +1,5 @@ /* -Copyright 2019 New Vector Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 37167cf600..f2c3c5c433 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -1,5 +1,5 @@ /* -Copyright 2019 New Vector Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/Searching.js b/src/Searching.js index ee46a66fb8..cb641ec72a 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -1,5 +1,5 @@ /* -Copyright 2019 New Vector Ltd +Copyright 2019 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 9fa8e8238a8cc1e406ed2e6ef471bfb452d707c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:18:09 +0100 Subject: [PATCH 39/64] BaseEventIndexManager: Fix a typo. --- src/BaseEventIndexManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index 7cefb023d1..61c556a0ff 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -76,7 +76,7 @@ export default class BaseEventIndexManager { /** * Does our EventIndexManager support event indexing. * - * If an EventIndexManager imlpementor has runtime dependencies that + * If an EventIndexManager implementor has runtime dependencies that * optionally enable event indexing they may override this method to perform * the necessary runtime checks here. * From 5149164010f1237e9e07e3a1a03b5cc0738e9b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:23:04 +0100 Subject: [PATCH 40/64] MatrixChat: Revert the unnecessary changes in the MatrixChat class. --- src/components/structures/MatrixChat.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index f78bb5c168..b45884e64f 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -1222,7 +1222,7 @@ export default createReactClass({ /** * Called when the session is logged out */ - _onLoggedOut: async function() { + _onLoggedOut: function() { this.notifyNewScreen('login'); this.setStateForNewView({ view: VIEWS.LOGIN, @@ -1272,9 +1272,8 @@ export default createReactClass({ // particularly noticeable when there are lots of 'limited' /sync responses // such as when laptops unsleep. // https://github.com/vector-im/riot-web/issues/3307#issuecomment-282895568 - cli.setCanResetTimelineCallback(async function(roomId) { + cli.setCanResetTimelineCallback(function(roomId) { console.log("Request to reset timeline in room ", roomId, " viewing:", self.state.currentRoomId); - if (roomId !== self.state.currentRoomId) { // It is safe to remove events from rooms we are not viewing. return true; From 910c3ac08db4bbdf8097e998cb486e6cdfef1a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:26:17 +0100 Subject: [PATCH 41/64] BaseEventIndexManager: Fix some type annotations. --- src/BaseEventIndexManager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BaseEventIndexManager.js b/src/BaseEventIndexManager.js index 61c556a0ff..5e8ca668ad 100644 --- a/src/BaseEventIndexManager.js +++ b/src/BaseEventIndexManager.js @@ -159,8 +159,8 @@ export default class BaseEventIndexManager { */ async addHistoricEvents( events: [HistoricEvent], - checkpoint: CrawlerCheckpoint | null = null, - oldCheckpoint: CrawlerCheckpoint | null = null, + checkpoint: CrawlerCheckpoint | null, + oldCheckpoint: CrawlerCheckpoint | null, ): Promise { throw new Error("Unimplemented"); } From ddb536e94a69485360611458cf70341720a3f604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:27:10 +0100 Subject: [PATCH 42/64] EventIndexPeg: Move a docstring to the correct place. --- src/EventIndexPeg.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 54d9c40079..4d0e518ab8 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -30,7 +30,8 @@ class EventIndexPeg { this.index = null; } - /** Create a new EventIndex and initialize it if the platform supports it. + /** + * Create a new EventIndex and initialize it if the platform supports it. * * @return {Promise} A promise that will resolve to true if an * EventIndex was successfully initialized, false otherwise. From 050e52ce461de709c01b1697379e6f8ad7fac18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:34:48 +0100 Subject: [PATCH 43/64] EventIndexPeg: Treat both cases of unavailable platform support the same. --- src/EventIndexPeg.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 4d0e518ab8..f1841b3f2b 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -38,9 +38,7 @@ class EventIndexPeg { */ async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return false; - - if (await indexManager.supportsEventIndexing() !== true) { + if (!indexManager || await indexManager.supportsEventIndexing() !== true) { console.log("EventIndex: Platform doesn't support event indexing,", "not initializing."); return false; From 3b06c684d23fd4e5cd012ccc197926b612fef63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:35:57 +0100 Subject: [PATCH 44/64] EventIndexing: Don't capitalize homeserver. --- src/EventIndexing.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index f2c3c5c433..05d5fd03da 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -59,8 +59,8 @@ export default class EventIndexer { return client.isRoomEncrypted(room.roomId); }; - // We only care to crawl the encrypted rooms, non-encrytped - // rooms can use the search provided by the Homeserver. + // We only care to crawl the encrypted rooms, non-encrypted. + // rooms can use the search provided by the homeserver. const encryptedRooms = rooms.filter(isRoomEncrypted); console.log("EventIndex: Adding initial crawler checkpoints"); @@ -189,7 +189,7 @@ export default class EventIndexer { while (!cancelled) { // This is a low priority task and we don't want to spam our - // Homeserver with /messages requests so we set a hefty timeout + // homeserver with /messages requests so we set a hefty timeout // here. await sleep(this._crawlerTimeout); @@ -210,7 +210,7 @@ export default class EventIndexer { console.log("EventIndex: crawling using checkpoint", checkpoint); // We have a checkpoint, let us fetch some messages, again, very - // conservatively to not bother our Homeserver too much. + // conservatively to not bother our homeserver too much. const eventMapper = client.getEventMapper(); // TODO we need to ensure to use member lazy loading with this // request so we get the correct profiles. From b4a6123295c896b49952302e4ced6ce166034419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:48:18 +0100 Subject: [PATCH 45/64] Searching: Move a comment to the correct place. --- src/Searching.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Searching.js b/src/Searching.js index cb641ec72a..4e6c8b9b4d 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -20,8 +20,9 @@ import MatrixClientPeg from "./MatrixClientPeg"; function serverSideSearch(term, roomId = undefined) { let filter; if (roomId !== undefined) { + // XXX: it's unintuitive that the filter for searching doesn't have + // the same shape as the v2 filter API :( filter = { - // XXX: it's unintuitive that the filter for searching doesn't have the same shape as the v2 filter API :( rooms: [roomId], }; } From a4ad8151f8415bf76bd1fd16b64ee167cc1bec0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:55:02 +0100 Subject: [PATCH 46/64] Searching: Use the short form to build the search arguments object. --- src/Searching.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index 4e6c8b9b4d..eb7137e221 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -28,8 +28,8 @@ function serverSideSearch(term, roomId = undefined) { } const searchPromise = MatrixClientPeg.get().searchRoomEvents({ - filter: filter, - term: term, + filter, + term, }); return searchPromise; From 0e3a0008df387bb036867177bb92451702f3fff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:56:57 +0100 Subject: [PATCH 47/64] Searching: Remove the func suffix from our search functions. --- src/Searching.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Searching.js b/src/Searching.js index eb7137e221..601da56f86 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -35,11 +35,11 @@ function serverSideSearch(term, roomId = undefined) { return searchPromise; } -async function combinedSearchFunc(searchTerm) { +async function combinedSearch(searchTerm) { // Create two promises, one for the local search, one for the // server-side search. const serverSidePromise = serverSideSearch(searchTerm); - const localPromise = localSearchFunc(searchTerm); + const localPromise = localSearch(searchTerm); // Wait for both promises to resolve. await Promise.all([serverSidePromise, localPromise]); @@ -74,7 +74,7 @@ async function combinedSearchFunc(searchTerm) { return result; } -async function localSearchFunc(searchTerm, roomId = undefined) { +async function localSearch(searchTerm, roomId = undefined) { const searchArgs = { search_term: searchTerm, before_limit: 1, @@ -115,7 +115,7 @@ function eventIndexSearch(term, roomId = undefined) { if (MatrixClientPeg.get().isRoomEncrypted(roomId)) { // The search is for a single encrypted room, use our local // search method. - searchPromise = localSearchFunc(term, roomId); + searchPromise = localSearch(term, roomId); } else { // The search is for a single non-encrypted room, use the // server-side search. @@ -124,7 +124,7 @@ function eventIndexSearch(term, roomId = undefined) { } else { // Search across all rooms, combine a server side search and a // local search. - searchPromise = combinedSearchFunc(term); + searchPromise = combinedSearch(term); } return searchPromise; From 2bb331cdf0c635728d2a08f993e2cb186a89e381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 10:57:23 +0100 Subject: [PATCH 48/64] Searching: Fix a typo. --- src/Searching.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Searching.js b/src/Searching.js index 601da56f86..ca3e7f041f 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -52,7 +52,7 @@ async function combinedSearch(searchTerm) { const result = {}; // Our localResult and serverSideResult are both ordered by - // recency separetly, when we combine them the order might not + // recency separately, when we combine them the order might not // be the right one so we need to sort them. const compare = (a, b) => { const aEvent = a.context.getEvent().event; From 579cbef7b0ed38f298fb35ad82b3d73096f080f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 14:29:03 +0100 Subject: [PATCH 49/64] EventIndexPeg: Rewrite the module documentation. --- src/EventIndexPeg.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index f1841b3f2b..a289c9e629 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -15,11 +15,8 @@ limitations under the License. */ /* - * Holds the current Platform object used by the code to do anything - * specific to the platform we're running on (eg. web, electron) - * Platforms are provided by the app layer. - * This allows the app layer to set a Platform without necessarily - * having to have a MatrixChat object + * Object holding the global EventIndex object. Can only be initialized if the + * platform supports event indexing. */ import PlatformPeg from "./PlatformPeg"; From 45e7aab41e3767026aa1e207640850f935f2aacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 14:30:07 +0100 Subject: [PATCH 50/64] EventIndexing: Rename our EventIndexer class. --- src/EventIndexing.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventIndexing.js b/src/EventIndexing.js index 05d5fd03da..38d610bac7 100644 --- a/src/EventIndexing.js +++ b/src/EventIndexing.js @@ -20,7 +20,7 @@ import MatrixClientPeg from "./MatrixClientPeg"; /** * Event indexing class that wraps the platform specific event indexing. */ -export default class EventIndexer { +export default class EventIndex { constructor() { this.crawlerCheckpoints = []; // The time that the crawler will wait between /rooms/{room_id}/messages From b983eaa3f9321a245c0f2f63d16a153dc13c9b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 14:36:08 +0100 Subject: [PATCH 51/64] EventIndex: Rename the file to be consistent with the class. --- src/{EventIndexing.js => EventIndex.js} | 0 src/EventIndexPeg.js | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{EventIndexing.js => EventIndex.js} (100%) diff --git a/src/EventIndexing.js b/src/EventIndex.js similarity index 100% rename from src/EventIndexing.js rename to src/EventIndex.js diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index a289c9e629..7530dd1a99 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -20,7 +20,7 @@ limitations under the License. */ import PlatformPeg from "./PlatformPeg"; -import EventIndex from "./EventIndexing"; +import EventIndex from "./EventIndex"; class EventIndexPeg { constructor() { From c48ccf9761d1f481da21b661bf88cbebef04da0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 14:40:04 +0100 Subject: [PATCH 52/64] EventIndex: Remove some unnecessary checks if event indexing is supported. --- src/EventIndex.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index 38d610bac7..e7aee6189e 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -40,7 +40,6 @@ export default class EventIndex { async onSync(state, prevState, data) { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return; if (prevState === null && state === "PREPARED") { // Load our stored checkpoints, if any. @@ -115,7 +114,6 @@ export default class EventIndex { async onRoomTimeline(ev, room, toStartOfTimeline, removed, data) { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return; // We only index encrypted rooms locally. if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -141,7 +139,6 @@ export default class EventIndex { async onEventDecrypted(ev, err) { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return; const eventId = ev.getId(); @@ -153,7 +150,6 @@ export default class EventIndex { async addLiveEventToIndex(ev) { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return; if (["m.room.message", "m.room.name", "m.room.topic"] .indexOf(ev.getType()) == -1) { @@ -349,7 +345,6 @@ export default class EventIndex { async onLimitedTimeline(room) { const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager === null) return; if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; const timeline = room.getLiveTimeline(); From 8d7e7d0cc404c8d2df9d9602a5f526e3bb01924b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 14:40:38 +0100 Subject: [PATCH 53/64] EventIndex: Remove the unused deleteEventIndex method. We need to support the deletion of the event index even if it's not currently initialized, therefore the deletion ended up in the EventIndexPeg class. --- src/EventIndex.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index e7aee6189e..6d8f265661 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -386,14 +386,6 @@ export default class EventIndex { return indexManager.closeEventIndex(); } - async deleteEventIndex() { - const indexManager = PlatformPeg.get().getEventIndexingManager(); - if (indexManager !== null) { - this.stopCrawler(); - await indexManager.deleteEventIndex(); - } - } - async search(searchArgs) { const indexManager = PlatformPeg.get().getEventIndexingManager(); return indexManager.searchEventIndex(searchArgs); From 4a6623bc00f9047256daaec382e3386b8f83741c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 15:04:22 +0100 Subject: [PATCH 54/64] EventIndex: Rework the crawler cancellation. --- src/EventIndex.js | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index 6d8f265661..75e3cda4f2 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -29,7 +29,7 @@ export default class EventIndex { // The maximum number of events our crawler should fetch in a single // crawl. this._eventsPerCrawl = 100; - this._crawlerRef = null; + this._crawler = null; this.liveEventsForIndex = new Set(); } @@ -165,7 +165,7 @@ export default class EventIndex { indexManager.addEventToIndex(e, profile); } - async crawlerFunc(handle) { + async crawlerFunc() { // TODO either put this in a better place or find a library provided // method that does this. const sleep = async (ms) => { @@ -179,7 +179,9 @@ export default class EventIndex { const client = MatrixClientPeg.get(); const indexManager = PlatformPeg.get().getEventIndexingManager(); - handle.cancel = () => { + this._crawler = {}; + + this._crawler.cancel = () => { cancelled = true; }; @@ -340,6 +342,8 @@ export default class EventIndex { } } + this._crawler = null; + console.log("EventIndex: Stopping crawler function"); } @@ -366,18 +370,13 @@ export default class EventIndex { } startCrawler() { - if (this._crawlerRef !== null) return; - - const crawlerHandle = {}; - this.crawlerFunc(crawlerHandle); - this._crawlerRef = crawlerHandle; + if (this._crawler !== null) return; + this.crawlerFunc(); } stopCrawler() { - if (this._crawlerRef === null) return; - - this._crawlerRef.cancel(); - this._crawlerRef = null; + if (this._crawler === null) return; + this._crawler.cancel(); } async close() { From 21f00aaeb1c6c47f314c9aed1543b3ba41208811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Mon, 18 Nov 2019 15:04:44 +0100 Subject: [PATCH 55/64] EventIndex: Fix some spelling errors. --- src/EventIndex.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index 75e3cda4f2..c96fe25fc8 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -282,8 +282,8 @@ export default class EventIndex { // attributes? }; - // TODO if there ar no events at this point we're missing a lot - // decryption keys, do we wan't to retry this checkpoint at a later + // TODO if there are no events at this point we're missing a lot + // decryption keys, do we want to retry this checkpoint at a later // stage? const filteredEvents = matrixEvents.filter(isValidEvent); @@ -336,7 +336,7 @@ export default class EventIndex { } } catch (e) { console.log("EventIndex: Error durring a crawl", e); - // An error occured, put the checkpoint back so we + // An error occurred, put the checkpoint back so we // can retry. this.crawlerCheckpoints.push(checkpoint); } From d2a99183595df253cdf4d97eee9c494e890fc4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 09:26:46 +0100 Subject: [PATCH 56/64] EventIndex: Remove some unused variables and some trailing whitespace. --- src/EventIndex.js | 4 ---- src/EventIndexPeg.js | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index c96fe25fc8..7ed43ad31c 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -113,8 +113,6 @@ export default class EventIndex { } async onRoomTimeline(ev, room, toStartOfTimeline, removed, data) { - const indexManager = PlatformPeg.get().getEventIndexingManager(); - // We only index encrypted rooms locally. if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -138,8 +136,6 @@ export default class EventIndex { } async onEventDecrypted(ev, err) { - const indexManager = PlatformPeg.get().getEventIndexingManager(); - const eventId = ev.getId(); // If the event isn't in our live event set, ignore it. diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 7530dd1a99..266b8f2d53 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -15,7 +15,7 @@ limitations under the License. */ /* - * Object holding the global EventIndex object. Can only be initialized if the + * Object holding the global EventIndex object. Can only be initialized if the * platform supports event indexing. */ From 6017473caf8e23e31666924a17fb5f6ce60af42d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 10:46:18 +0100 Subject: [PATCH 57/64] EventIndex: Move the event listener registration into the EventIndex class. --- src/EventIndex.js | 41 +++++++++++++++++++++++-- src/EventIndexPeg.js | 3 +- src/components/structures/MatrixChat.js | 26 ---------------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/EventIndex.js b/src/EventIndex.js index 7ed43ad31c..b6784cd331 100644 --- a/src/EventIndex.js +++ b/src/EventIndex.js @@ -31,11 +31,45 @@ export default class EventIndex { this._eventsPerCrawl = 100; this._crawler = null; this.liveEventsForIndex = new Set(); + + this.boundOnSync = async (state, prevState, data) => { + await this.onSync(state, prevState, data); + }; + this.boundOnRoomTimeline = async ( ev, room, toStartOfTimeline, removed, + data) => { + await this.onRoomTimeline(ev, room, toStartOfTimeline, removed, data); + }; + this.boundOnEventDecrypted = async (ev, err) => { + await this.onEventDecrypted(ev, err); + }; + this.boundOnTimelineReset = async (room, timelineSet, + resetAllTimelines) => await this.onTimelineReset(room); } async init() { const indexManager = PlatformPeg.get().getEventIndexingManager(); - return indexManager.initEventIndex(); + await indexManager.initEventIndex(); + + this.registerListeners(); + } + + registerListeners() { + const client = MatrixClientPeg.get(); + + client.on('sync', this.boundOnSync); + client.on('Room.timeline', this.boundOnRoomTimeline); + client.on('Event.decrypted', this.boundOnEventDecrypted); + client.on('Room.timelineReset', this.boundOnTimelineReset); + } + + removeListeners() { + const client = MatrixClientPeg.get(); + if (client === null) return; + + client.removeListener('sync', this.boundOnSync); + client.removeListener('Room.timeline', this.boundOnRoomTimeline); + client.removeListener('Event.decrypted', this.boundOnEventDecrypted); + client.removeListener('Room.timelineReset', this.boundOnTimelineReset); } async onSync(state, prevState, data) { @@ -343,7 +377,9 @@ export default class EventIndex { console.log("EventIndex: Stopping crawler function"); } - async onLimitedTimeline(room) { + async onTimelineReset(room) { + if (room === null) return; + const indexManager = PlatformPeg.get().getEventIndexingManager(); if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -377,6 +413,7 @@ export default class EventIndex { async close() { const indexManager = PlatformPeg.get().getEventIndexingManager(); + this.removeListeners(); this.stopCrawler(); return indexManager.closeEventIndex(); } diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 266b8f2d53..74b7968c70 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -97,10 +97,9 @@ class EventIndexPeg { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (indexManager !== null) { - this.stop(); + this.unset(); console.log("EventIndex: Deleting event index."); await indexManager.deleteEventIndex(); - this.index = null; } } } diff --git a/src/components/structures/MatrixChat.js b/src/components/structures/MatrixChat.js index b45884e64f..da67416400 100644 --- a/src/components/structures/MatrixChat.js +++ b/src/components/structures/MatrixChat.js @@ -31,7 +31,6 @@ import Analytics from "../../Analytics"; import { DecryptionFailureTracker } from "../../DecryptionFailureTracker"; import MatrixClientPeg from "../../MatrixClientPeg"; import PlatformPeg from "../../PlatformPeg"; -import EventIndexPeg from "../../EventIndexPeg"; import SdkConfig from "../../SdkConfig"; import * as RoomListSorter from "../../RoomListSorter"; import dis from "../../dispatcher"; @@ -1288,31 +1287,6 @@ export default createReactClass({ return self._loggedInView.child.canResetTimelineInRoom(roomId); }); - cli.on('sync', async (state, prevState, data) => { - const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return; - await eventIndex.onSync(state, prevState, data); - }); - - cli.on("Room.timeline", async (ev, room, toStartOfTimeline, removed, data) => { - const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return; - await eventIndex.onRoomTimeline(ev, room, toStartOfTimeline, removed, data); - }); - - cli.on("Event.decrypted", async (ev, err) => { - const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return; - await eventIndex.onEventDecrypted(ev, err); - }); - - cli.on("Room.timelineReset", async (room, timelineSet, resetAllTimelines) => { - const eventIndex = EventIndexPeg.get(); - if (eventIndex === null) return; - if (room === null) return; - await eventIndex.onLimitedTimeline(room); - }); - cli.on('sync', function(state, prevState, data) { // LifecycleStore and others cannot directly subscribe to matrix client for // events because flux only allows store state changes during flux dispatches. From 979803797fb58bb421c1e1a98cf90f263ae3af91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 11:05:37 +0100 Subject: [PATCH 58/64] Lifecycle: Make the clear storage method async. --- src/Lifecycle.js | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 1d38934ade..1b69ca6ade 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -607,20 +607,20 @@ async function startMatrixClient(startSyncing=true) { * Stops a running client and all related services, and clears persistent * storage. Used after a session has been logged out. */ -export function onLoggedOut() { +export async function onLoggedOut() { _isLoggingOut = false; // Ensure that we dispatch a view change **before** stopping the client so // so that React components unmount first. This avoids React soft crashes // that can occur when components try to use a null client. dis.dispatch({action: 'on_logged_out'}); stopMatrixClient(); - _clearStorage().done(); + await _clearStorage(); } /** * @returns {Promise} promise which resolves once the stores have been cleared */ -function _clearStorage() { +async function _clearStorage() { Analytics.logout(); if (window.localStorage) { @@ -633,12 +633,8 @@ function _clearStorage() { baseUrl: "", }); - const clear = async () => { - await EventIndexPeg.deleteEventIndex(); - await cli.clearStores(); - }; - - return clear(); + await EventIndexPeg.deleteEventIndex(); + await cli.clearStores(); } /** @@ -662,7 +658,7 @@ export function stopMatrixClient(unsetClient=true) { if (unsetClient) { MatrixClientPeg.unset(); - EventIndexPeg.unset().done(); + EventIndexPeg.unset(); } } } From f776bdcc8b4306557df2bda71bce6ec097694abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 12:23:49 +0100 Subject: [PATCH 59/64] EventIndex: Hide the feature behind a labs flag. --- src/EventIndexPeg.js | 5 +++++ src/i18n/strings/en_EN.json | 3 ++- src/settings/Settings.js | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/EventIndexPeg.js b/src/EventIndexPeg.js index 74b7968c70..eb4caa2ca4 100644 --- a/src/EventIndexPeg.js +++ b/src/EventIndexPeg.js @@ -21,6 +21,7 @@ limitations under the License. import PlatformPeg from "./PlatformPeg"; import EventIndex from "./EventIndex"; +import SettingsStore from './settings/SettingsStore'; class EventIndexPeg { constructor() { @@ -34,6 +35,10 @@ class EventIndexPeg { * EventIndex was successfully initialized, false otherwise. */ async init() { + if (!SettingsStore.isFeatureEnabled("feature_event_indexing")) { + return false; + } + const indexManager = PlatformPeg.get().getEventIndexingManager(); if (!indexManager || await indexManager.supportsEventIndexing() !== true) { console.log("EventIndex: Platform doesn't support event indexing,", diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index f524a22d4b..69c3f07f3f 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -1829,5 +1829,6 @@ "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.": "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.", "Failed to set direct chat tag": "Failed to set direct chat tag", "Failed to remove tag %(tagName)s from room": "Failed to remove tag %(tagName)s from room", - "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room" + "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room", + "Enable local event indexing and E2EE search (requires restart)": "Enable local event indexing and E2EE search (requires restart)" } diff --git a/src/settings/Settings.js b/src/settings/Settings.js index 3c33ae57fe..8abd845f0c 100644 --- a/src/settings/Settings.js +++ b/src/settings/Settings.js @@ -120,6 +120,12 @@ export const SETTINGS = { supportedLevels: LEVELS_FEATURE, default: false, }, + "feature_event_indexing": { + isFeature: true, + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + displayName: _td("Enable local event indexing and E2EE search (requires restart)"), + default: false, + }, "useCiderComposer": { displayName: _td("Use the new, faster, composer for writing messages"), supportedLevels: LEVELS_ACCOUNT_SETTINGS, From e9df973c8273f7a0958a69e257cd2d9204ce8404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 12:52:12 +0100 Subject: [PATCH 60/64] EventIndex: Move the event indexing files into a separate folder. --- src/BasePlatform.js | 2 +- src/Lifecycle.js | 2 +- src/Searching.js | 2 +- src/{ => indexing}/BaseEventIndexManager.js | 0 src/{ => indexing}/EventIndex.js | 4 ++-- src/{ => indexing}/EventIndexPeg.js | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) rename src/{ => indexing}/BaseEventIndexManager.js (100%) rename src/{ => indexing}/EventIndex.js (99%) rename src/{ => indexing}/EventIndexPeg.js (95%) diff --git a/src/BasePlatform.js b/src/BasePlatform.js index f6301fd173..14e34a1f40 100644 --- a/src/BasePlatform.js +++ b/src/BasePlatform.js @@ -19,7 +19,7 @@ limitations under the License. */ import dis from './dispatcher'; -import BaseEventIndexManager from './BaseEventIndexManager'; +import BaseEventIndexManager from './indexing/BaseEventIndexManager'; /** * Base class for classes that provide platform-specific functionality diff --git a/src/Lifecycle.js b/src/Lifecycle.js index 1b69ca6ade..65fa0b29ce 100644 --- a/src/Lifecycle.js +++ b/src/Lifecycle.js @@ -20,7 +20,7 @@ import Promise from 'bluebird'; import Matrix from 'matrix-js-sdk'; import MatrixClientPeg from './MatrixClientPeg'; -import EventIndexPeg from './EventIndexPeg'; +import EventIndexPeg from './indexing/EventIndexPeg'; import createMatrixClient from './utils/createMatrixClient'; import Analytics from './Analytics'; import Notifier from './Notifier'; diff --git a/src/Searching.js b/src/Searching.js index ca3e7f041f..f8976c92e4 100644 --- a/src/Searching.js +++ b/src/Searching.js @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import EventIndexPeg from "./EventIndexPeg"; +import EventIndexPeg from "./indexing/EventIndexPeg"; import MatrixClientPeg from "./MatrixClientPeg"; function serverSideSearch(term, roomId = undefined) { diff --git a/src/BaseEventIndexManager.js b/src/indexing/BaseEventIndexManager.js similarity index 100% rename from src/BaseEventIndexManager.js rename to src/indexing/BaseEventIndexManager.js diff --git a/src/EventIndex.js b/src/indexing/EventIndex.js similarity index 99% rename from src/EventIndex.js rename to src/indexing/EventIndex.js index b6784cd331..df81667c6e 100644 --- a/src/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import PlatformPeg from "./PlatformPeg"; -import MatrixClientPeg from "./MatrixClientPeg"; +import PlatformPeg from "../PlatformPeg"; +import MatrixClientPeg from "../MatrixClientPeg"; /** * Event indexing class that wraps the platform specific event indexing. diff --git a/src/EventIndexPeg.js b/src/indexing/EventIndexPeg.js similarity index 95% rename from src/EventIndexPeg.js rename to src/indexing/EventIndexPeg.js index eb4caa2ca4..c0bdd74ff4 100644 --- a/src/EventIndexPeg.js +++ b/src/indexing/EventIndexPeg.js @@ -19,9 +19,9 @@ limitations under the License. * platform supports event indexing. */ -import PlatformPeg from "./PlatformPeg"; -import EventIndex from "./EventIndex"; -import SettingsStore from './settings/SettingsStore'; +import PlatformPeg from "../PlatformPeg"; +import EventIndex from "../indexing/EventIndex"; +import SettingsStore from '../settings/SettingsStore'; class EventIndexPeg { constructor() { From 27d1e4fbbedb6ccf1bccdf17bd1cee74dee4c224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Tue, 19 Nov 2019 14:17:51 +0100 Subject: [PATCH 61/64] Fix the translations en_EN file by regenerating it. --- src/i18n/strings/en_EN.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 69c3f07f3f..6f116cbac2 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -334,6 +334,7 @@ "Group & filter rooms by custom tags (refresh to apply changes)": "Group & filter rooms by custom tags (refresh to apply changes)", "Render simple counters in room header": "Render simple counters in room header", "Multiple integration managers": "Multiple integration managers", + "Enable local event indexing and E2EE search (requires restart)": "Enable local event indexing and E2EE search (requires restart)", "Use the new, faster, composer for writing messages": "Use the new, faster, composer for writing messages", "Enable Emoji suggestions while typing": "Enable Emoji suggestions while typing", "Use compact timeline layout": "Use compact timeline layout", @@ -1829,6 +1830,5 @@ "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.": "If you didn't remove the recovery method, an attacker may be trying to access your account. Change your account password and set a new recovery method immediately in Settings.", "Failed to set direct chat tag": "Failed to set direct chat tag", "Failed to remove tag %(tagName)s from room": "Failed to remove tag %(tagName)s from room", - "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room", - "Enable local event indexing and E2EE search (requires restart)": "Enable local event indexing and E2EE search (requires restart)" + "Failed to add tag %(tagName)s to room": "Failed to add tag %(tagName)s to room" } From 2f5b0a9652629cb75bdf39926ff2f045511286ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 20 Nov 2019 12:30:03 +0100 Subject: [PATCH 62/64] EventIndex: Use property initializer style for the bound callbacks. --- src/indexing/EventIndex.js | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index df81667c6e..e6a1d4007b 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -31,19 +31,6 @@ export default class EventIndex { this._eventsPerCrawl = 100; this._crawler = null; this.liveEventsForIndex = new Set(); - - this.boundOnSync = async (state, prevState, data) => { - await this.onSync(state, prevState, data); - }; - this.boundOnRoomTimeline = async ( ev, room, toStartOfTimeline, removed, - data) => { - await this.onRoomTimeline(ev, room, toStartOfTimeline, removed, data); - }; - this.boundOnEventDecrypted = async (ev, err) => { - await this.onEventDecrypted(ev, err); - }; - this.boundOnTimelineReset = async (room, timelineSet, - resetAllTimelines) => await this.onTimelineReset(room); } async init() { @@ -56,23 +43,23 @@ export default class EventIndex { registerListeners() { const client = MatrixClientPeg.get(); - client.on('sync', this.boundOnSync); - client.on('Room.timeline', this.boundOnRoomTimeline); - client.on('Event.decrypted', this.boundOnEventDecrypted); - client.on('Room.timelineReset', this.boundOnTimelineReset); + client.on('sync', this.onSync); + client.on('Room.timeline', this.onRoomTimeline); + client.on('Event.decrypted', this.onEventDecrypted); + client.on('Room.timelineReset', this.onTimelineReset); } removeListeners() { const client = MatrixClientPeg.get(); if (client === null) return; - client.removeListener('sync', this.boundOnSync); - client.removeListener('Room.timeline', this.boundOnRoomTimeline); - client.removeListener('Event.decrypted', this.boundOnEventDecrypted); - client.removeListener('Room.timelineReset', this.boundOnTimelineReset); + client.removeListener('sync', this.onSync); + client.removeListener('Room.timeline', this.onRoomTimeline); + client.removeListener('Event.decrypted', this.onEventDecrypted); + client.removeListener('Room.timelineReset', this.onTimelineReset); } - async onSync(state, prevState, data) { + onSync = async (state, prevState, data) => { const indexManager = PlatformPeg.get().getEventIndexingManager(); if (prevState === null && state === "PREPARED") { @@ -146,7 +133,7 @@ export default class EventIndex { } } - async onRoomTimeline(ev, room, toStartOfTimeline, removed, data) { + onRoomTimeline = async (ev, room, toStartOfTimeline, removed, data) => { // We only index encrypted rooms locally. if (!MatrixClientPeg.get().isRoomEncrypted(room.roomId)) return; @@ -169,7 +156,7 @@ export default class EventIndex { } } - async onEventDecrypted(ev, err) { + onEventDecrypted = async (ev, err) => { const eventId = ev.getId(); // If the event isn't in our live event set, ignore it. @@ -377,7 +364,7 @@ export default class EventIndex { console.log("EventIndex: Stopping crawler function"); } - async onTimelineReset(room) { + onTimelineReset = async (room, timelineSet, resetAllTimelines) => { if (room === null) return; const indexManager = PlatformPeg.get().getEventIndexingManager(); From 0631faf902c5870b263d8f2745745c6ae2281a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 20 Nov 2019 12:31:07 +0100 Subject: [PATCH 63/64] Settings: Fix the supportedLevels for event indexing feature. --- src/settings/Settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings/Settings.js b/src/settings/Settings.js index 8abd845f0c..2cf9509aca 100644 --- a/src/settings/Settings.js +++ b/src/settings/Settings.js @@ -122,7 +122,7 @@ export const SETTINGS = { }, "feature_event_indexing": { isFeature: true, - supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + supportedLevels: LEVELS_FEATURE, displayName: _td("Enable local event indexing and E2EE search (requires restart)"), default: false, }, From 4bd46f9d694f03aeaad667e35ab64083f7d4479f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damir=20Jeli=C4=87?= Date: Wed, 20 Nov 2019 12:47:20 +0100 Subject: [PATCH 64/64] EventIndex: Silence the linter complaining about missing docs. --- src/indexing/EventIndex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indexing/EventIndex.js b/src/indexing/EventIndex.js index e6a1d4007b..6bad992017 100644 --- a/src/indexing/EventIndex.js +++ b/src/indexing/EventIndex.js @@ -17,7 +17,7 @@ limitations under the License. import PlatformPeg from "../PlatformPeg"; import MatrixClientPeg from "../MatrixClientPeg"; -/** +/* * Event indexing class that wraps the platform specific event indexing. */ export default class EventIndex {