diff --git a/src/client.js b/src/client.js
index 12bfc6a33..b084c21fc 100644
--- a/src/client.js
+++ b/src/client.js
@@ -353,6 +353,10 @@ export function MatrixClient(opts) {
     if (call) {
         this._callEventHandler = new CallEventHandler(this);
         this._supportsVoip = true;
+        // Start listening for calls after the initial sync is done
+        // We do not need to backfill the call event buffer
+        // with encrypted events that might never get decrypted
+        this.once("sync", () => this._callEventHandler.start());
     } else {
         this._callEventHandler = null;
     }
diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts
index 9c4c38b26..0d35f55e3 100644
--- a/src/webrtc/callEventHandler.ts
+++ b/src/webrtc/callEventHandler.ts
@@ -43,6 +43,9 @@ export class CallEventHandler {
         // after loading and after we've been offline for a bit.
         this.callEventBuffer = [];
         this.candidateEventsByCall = new Map<string, Array<MatrixEvent>>();
+    }
+
+    public start() {
         this.client.on("sync", this.evaluateEventBuffer);
         this.client.on("event", this.onEvent);
     }
@@ -85,37 +88,16 @@ export class CallEventHandler {
         }
     }
 
-    private onEvent = (event: MatrixEvent) => {
-        // any call events or ones that might be once they're decrypted
+    private onEvent = async (event: MatrixEvent) => {
+        await this.client.decryptEventIfNeeded(event);
         if (
             event.getType().indexOf("m.call.") === 0 ||
             event.getType().indexOf("org.matrix.call.") === 0
-            || event.isBeingDecrypted()
         ) {
             // queue up for processing once all events from this sync have been
             // processed (see above).
             this.callEventBuffer.push(event);
         }
-
-        if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
-            // add an event listener for once the event is decrypted.
-            event.once("Event.decrypted", () => {
-                if (event.getType().indexOf("m.call.") === -1) return;
-
-                if (this.callEventBuffer.includes(event)) {
-                    // we were waiting for that event to decrypt, so recheck the buffer
-                    this.evaluateEventBuffer();
-                } else {
-                    // This one wasn't buffered so just run the event handler for it
-                    // straight away
-                    try {
-                        this.handleCallEvent(event);
-                    } catch (e) {
-                        logger.error("Caught exception handling call event", e);
-                    }
-                }
-            });
-        }
     }
 
     private handleCallEvent(event: MatrixEvent) {