diff --git a/spec/integ/devicelist-integ-spec.js b/spec/integ/devicelist-integ-spec.js
index 9ba53783c..8e0bd1bd4 100644
--- a/spec/integ/devicelist-integ-spec.js
+++ b/spec/integ/devicelist-integ-spec.js
@@ -366,9 +366,9 @@ describe("DeviceList management:", function() {
                 anotherTestClient.httpBackend.when('GET', '/sync').respond(
                     200, getSyncResponse([]));
                 await anotherTestClient.flushSync();
-                await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+                await anotherTestClient.client._crypto._deviceList.saveIfDirty();
 
-                aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
                     const bobStat = data.trackingStatus['@bob:xyz'];
 
                     expect(bobStat).toEqual(
diff --git a/src/crypto/DeviceList.js b/src/crypto/DeviceList.js
index 61661f3a5..2ed10449f 100644
--- a/src/crypto/DeviceList.js
+++ b/src/crypto/DeviceList.js
@@ -417,6 +417,19 @@ export default class DeviceList {
         this._dirty = true;
     }
 
+    /**
+     * Set all users we're currently tracking to untracked
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    stopTrackingAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        console.log("stopped tracking all: "+JSON.stringify(this._deviceTrackingStatus));
+        this._dirty = true;
+    }
 
     /**
      * Mark the cached device list for the given user outdated.
@@ -440,18 +453,6 @@ export default class DeviceList {
         this._dirty = true;
     }
 
-    /**
-     * Mark all tracked device lists as outdated.
-     *
-     * This will flag each user whose devices we are tracking as in need of an
-     * update.
-     */
-    invalidateAllDeviceLists() {
-        for (const userId of Object.keys(this._deviceTrackingStatus)) {
-            this.invalidateUserDeviceList(userId);
-        }
-    }
-
     /**
      * If we have users who have outdated device lists, start key downloads for them
      *
diff --git a/src/crypto/index.js b/src/crypto/index.js
index 9d80544b2..114ac2126 100644
--- a/src/crypto/index.js
+++ b/src/crypto/index.js
@@ -894,6 +894,22 @@ Crypto.prototype.onCryptoEvent = async function(event) {
     }
 };
 
+/**
+ * Called before the result of a sync is procesed
+ *
+ * @param {Object} syncData  the data from the 'MatrixClient.sync' event
+ */
+Crypto.prototype.onSyncWillProcess = async function(syncData) {
+    if (!syncData.oldSyncToken) {
+        // If there is no old sync token, we start all our tracking from
+        // scratch, so mark everything as untracked. onCryptoEvent will
+        // be called for all e2e rooms during the processing of the sync,
+        // at which point we'll start tracking all the users of that room.
+        console.log("Initial sync performed - resetting device tracking state");
+        this._deviceList.stopTrackingAllDeviceLists();
+    }
+};
+
 /**
  * handle the completion of a /sync
  *
@@ -905,14 +921,6 @@ Crypto.prototype.onCryptoEvent = async function(event) {
 Crypto.prototype.onSyncCompleted = async function(syncData) {
     const nextSyncToken = syncData.nextSyncToken;
 
-    if (!syncData.oldSyncToken) {
-        // If we have a stored device sync token, we could request the complete
-        // list of device changes from the server here to get our device list up
-        // to date. This case should be relatively rare though (only when you hit
-        // 'clear cache and reload' in practice) so we just invalidate everything.
-        console.log("invalidating all device list caches after inital sync");
-        this._deviceList.invalidateAllDeviceLists();
-    }
     this._deviceList.setSyncToken(syncData.nextSyncToken);
     this._deviceList.saveIfDirty();
 
@@ -945,12 +953,32 @@ Crypto.prototype._evalDeviceListChanges = async function(deviceLists) {
     }
 
     if (deviceLists.left && Array.isArray(deviceLists.left)) {
+        const e2eUserIds = new Set(this._getE2eUsers());
+
         deviceLists.left.forEach((u) => {
-            this._deviceList.stopTrackingDeviceList(u);
+            if (!e2eUserIds.has(u)) {
+                this._deviceList.stopTrackingDeviceList(u);
+            }
         });
     }
 };
 
+/**
+ * Get a list of all the IDs of users we share an e2e room with
+ *
+ * @returns {string[]} List of user IDs
+ */
+Crypto.prototype._getE2eUsers = function() {
+    const e2eUserIds = [];
+    for (const room of this._getE2eRooms()) {
+        const members = room.getJoinedMembers();
+        for (const member of members) {
+            e2eUserIds.push(member.userId);
+        }
+    }
+    return e2eUserIds;
+};
+
 /**
  * Get a list of the e2e-enabled rooms we are members of
  *
diff --git a/src/sync.js b/src/sync.js
index 90973237c..05406b2fd 100644
--- a/src/sync.js
+++ b/src/sync.js
@@ -627,6 +627,12 @@ SyncApi.prototype._sync = async function(syncOptions) {
         catchingUp: this._catchingUp,
     };
 
+    if (this.opts.crypto && !isCachedResponse) {
+        // tell the crypto module we're about to process a sync
+        // response
+        await this.opts.crypto.onSyncWillProcess(syncEventData);
+    }
+
     try {
         await this._processSyncResponse(syncEventData, data, isCachedResponse);
     } catch(e) {