diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts
index e36153f3e..70f6c241b 100644
--- a/src/webrtc/call.ts
+++ b/src/webrtc/call.ts
@@ -1084,8 +1084,21 @@ export class MatrixCall extends EventEmitter {
             await this.peerConn.setRemoteDescription(description);
 
             if (description.type === 'offer') {
+                // First we sent the direction of the tranciever to what we'd like it to be,
+                // irresepective of whether the other side has us on hold - so just whether we
+                // want the call to be on hold or not. This is necessary because in a few lines,
+                // we'll adjust the direction and unless we do this too, we'll never come off hold.
+                for (const tranceiver of this.peerConn.getTransceivers()) {
+                    tranceiver.direction = this.isRemoteOnHold() ? 'inactive' : 'sendrecv';
+                }
                 const localDescription = await this.peerConn.createAnswer();
                 await this.peerConn.setLocalDescription(localDescription);
+                // Now we've got our answer, set the direction to the outcome of the negotiation.
+                // We need to do this otherwise Firefox will notice that the direction is not the
+                // currentDirection and try to negotiate itself off hold again.
+                for (const tranceiver of this.peerConn.getTransceivers()) {
+                    tranceiver.direction = tranceiver.currentDirection;
+                }
 
                 this.sendVoipEvent(EventType.CallNegotiate, {
                     description: this.peerConn.localDescription,