diff --git a/src/client.js b/src/client.js
index ea6e32e86..e8b8c1cf8 100644
--- a/src/client.js
+++ b/src/client.js
@@ -242,19 +242,32 @@ function MatrixClient(opts) {
         const actions = this._pushProcessor.actionsForEvent(event);
         event.setPushActions(actions); // Might as well while we're here
 
+        const room = this.getRoom(event.getRoomId());
+        if (!room) return;
+
+        const currentCount = room.getUnreadNotificationCount("highlight");
+
         // Ensure the unread counts are kept up to date if the event is encrypted
+        // We also want to make sure that the notification count goes up if we already
+        // have encrypted events to avoid other code from resetting 'highlight' to zero.
         const oldHighlight = oldActions && oldActions.tweaks
             ? !!oldActions.tweaks.highlight : false;
         const newHighlight = actions && actions.tweaks
             ? !!actions.tweaks.highlight : false;
-        if (oldHighlight !== newHighlight) {
-            const room = this.getRoom(event.getRoomId());
+        if (oldHighlight !== newHighlight || currentCount > 0) {
             // TODO: Handle mentions received while the client is offline
             // See also https://github.com/vector-im/riot-web/issues/9069
-            if (room && !room.hasUserReadEvent(this.getUserId(), event.getId())) {
-                const current = room.getUnreadNotificationCount("highlight");
-                const newCount = newHighlight ? current + 1 : current - 1;
+            if (!room.hasUserReadEvent(this.getUserId(), event.getId())) {
+                let newCount = currentCount;
+                if (newHighlight && !oldHighlight) newCount++;
+                if (!newHighlight && oldHighlight) newCount--;
                 room.setUnreadNotificationCount("highlight", newCount);
+
+                // Fix 'Mentions Only' rooms from not having the right badge count
+                const totalCount = room.getUnreadNotificationCount('total');
+                if (totalCount < newCount) {
+                    room.setUnreadNotificationCount('total', newCount);
+                }
             }
         }
     });
diff --git a/src/sync.js b/src/sync.js
index 337f4100c..f024314e5 100644
--- a/src/sync.js
+++ b/src/sync.js
@@ -1076,9 +1076,17 @@ SyncApi.prototype._processSyncResponse = async function(
             room.setUnreadNotificationCount(
                 'total', joinObj.unread_notifications.notification_count,
             );
-            room.setUnreadNotificationCount(
-                'highlight', joinObj.unread_notifications.highlight_count,
-            );
+
+            // We track unread notifications ourselves in encrypted rooms, so don't
+            // bother setting it here. We trust our calculations better than the
+            // server's for this case, and therefore will assume that our non-zero
+            // count is accurate.
+            if (client.isRoomEncrypted(room.roomId)
+                && room.getUnreadNotificationCount('highlight') <= 0) {
+                room.setUnreadNotificationCount(
+                    'highlight', joinObj.unread_notifications.highlight_count,
+                );
+            }
         }
 
         room.updateMyMembership("join");