diff --git a/lib/crypto/algorithms/base.js b/lib/crypto/algorithms/base.js
index 6d25bc2bd..387ab448b 100644
--- a/lib/crypto/algorithms/base.js
+++ b/lib/crypto/algorithms/base.js
@@ -105,11 +105,16 @@ EncryptionAlgorithm.prototype.onNewDevice = function(userId, deviceId) {};
  *
  * @param {object} params parameters
  * @param {string} params.userId  The UserID for the local user
+ * @param {module:crypto} params.crypto crypto core
  * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
+ * @param {string=} params.roomId The ID of the room we will be receiving
+ *     from. Null for to-device events.
  */
 var DecryptionAlgorithm = function(params) {
     this._userId = params.userId;
+    this._crypto = params.crypto;
     this._olmDevice = params.olmDevice;
+    this._roomId = params.roomId;
 };
 /** */
 module.exports.DecryptionAlgorithm = DecryptionAlgorithm;
diff --git a/lib/crypto/index.js b/lib/crypto/index.js
index 2f0c15cc8..c444e4246 100644
--- a/lib/crypto/index.js
+++ b/lib/crypto/index.js
@@ -57,7 +57,10 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId) {
     this._olmDevice = new OlmDevice(sessionStore);
 
     // EncryptionAlgorithm instance for each room
-    this._roomAlgorithms = {};
+    this._roomEncryptors = {};
+
+    // map from algorithm to DecryptionAlgorithm instance, for each room
+    this._roomDecryptors = {};
 
     this._supportedAlgorithms = utils.keys(
         algorithms.DECRYPTION_CLASSES
@@ -705,7 +708,7 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
         roomId: roomId,
         config: config,
     });
-    this._roomAlgorithms[roomId] = alg;
+    this._roomEncryptors[roomId] = alg;
 };
 
 
@@ -839,7 +842,7 @@ Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
  * @return {bool} whether encryption is enabled.
  */
 Crypto.prototype.isRoomEncrypted = function(roomId) {
-    return Boolean(this._roomAlgorithms[roomId]);
+    return Boolean(this._roomEncryptors[roomId]);
 };
 
 /**
@@ -867,7 +870,7 @@ Crypto.prototype.encryptEventIfNeeded = function(event, room) {
 
     var roomId = event.getRoomId();
 
-    var alg = this._roomAlgorithms[roomId];
+    var alg = this._roomEncryptors[roomId];
     if (!alg) {
         // not encrypting messages in this room
 
@@ -922,14 +925,7 @@ Crypto.prototype.encryptEventIfNeeded = function(event, room) {
  */
 Crypto.prototype.decryptEvent = function(event) {
     var content = event.content;
-    var AlgClass = algorithms.DECRYPTION_CLASSES[content.algorithm];
-    if (!AlgClass) {
-        throw new algorithms.DecryptionError("Unable to decrypt " + content.algorithm);
-    }
-    var alg = new AlgClass({
-        userId: this._userId,
-        olmDevice: this._olmDevice,
-    });
+    var alg = this._getRoomDecryptor(event.room_id, content.algorithm);
     var r = alg.decryptEvent(event);
 
     if (r !== null) {
@@ -1054,7 +1050,7 @@ Crypto.prototype._onInitialSyncCompleted = function(rooms) {
         var room = rooms[i];
 
         // check for rooms with encryption enabled
-        var alg = this._roomAlgorithms[room.roomId];
+        var alg = this._roomEncryptors[room.roomId];
         if (!alg) {
             continue;
         }
@@ -1108,16 +1104,13 @@ Crypto.prototype._onInitialSyncCompleted = function(rooms) {
  */
 Crypto.prototype._onRoomKeyEvent = function(event) {
     var content = event.getContent();
-    var AlgClass = algorithms.DECRYPTION_CLASSES[content.algorithm];
-    if (!AlgClass) {
-        throw new algorithms.DecryptionError(
-            "Unable to handle keys for " + content.algorithm
-        );
+
+    if (!content.room_id || !content.algorithm) {
+        console.error("key event is missing fields");
+        return;
     }
-    var alg = new AlgClass({
-        userId: this._userId,
-        olmDevice: this._olmDevice,
-    });
+
+    var alg = this._getRoomDecryptor(content.room_id, content.algorithm);
     alg.onRoomKeyEvent(event);
 };
 
@@ -1141,7 +1134,7 @@ Crypto.prototype._onRoomMembership = function(event, member, oldMembership) {
 
     var roomId = member.roomId;
 
-    var alg = this._roomAlgorithms[roomId];
+    var alg = this._roomEncryptors[roomId];
     if (!alg) {
         // not encrypting in this room
         return;
@@ -1177,7 +1170,7 @@ Crypto.prototype._onNewDeviceEvent = function(event) {
     ).then(function() {
         for (var i = 0; i < rooms.length; i++) {
             var roomId = rooms[i];
-            var alg = self._roomAlgorithms[roomId];
+            var alg = self._roomEncryptors[roomId];
             if (!alg) {
                 // not encrypting in this room
                 continue;
@@ -1193,6 +1186,58 @@ Crypto.prototype._onNewDeviceEvent = function(event) {
     }).done();
 };
 
+/**
+ * Get a decryptor for a given room and algorithm.
+ *
+ * If we already have a decryptor for the given room and algorithm, return
+ * it. Otherwise try to instantiate it.
+ *
+ * @private
+ *
+ * @param {string?} roomId   room id for decryptor. If undefined, a temporary
+ * decryptor is instantiated.
+ *
+ * @param {string} algorithm  crypto algorithm
+ *
+ * @return {module:crypto.algorithms.base.DecryptionAlgorithm}
+ *
+ * @raises {module:crypto.algorithms.DecryptionError} if the algorithm is
+ * unknown
+ */
+Crypto.prototype._getRoomDecryptor = function(roomId, algorithm) {
+    var decryptors;
+    var alg;
+
+    roomId = roomId || null;
+    if (roomId) {
+        decryptors = this._roomDecryptors[roomId];
+        if (!decryptors) {
+            this._roomDecryptors[roomId] = decryptors = {};
+        }
+
+        alg = decryptors[algorithm];
+        if (alg) {
+            return alg;
+        }
+    }
+
+    var AlgClass = algorithms.DECRYPTION_CLASSES[algorithm];
+    if (!AlgClass) {
+        throw new algorithms.DecryptionError("Unable to decrypt " + algorithm);
+    }
+    alg = new AlgClass({
+        userId: this._userId,
+        crypto: this,
+        olmDevice: this._olmDevice,
+        roomId: roomId,
+    });
+
+    if (decryptors) {
+        decryptors[algorithm] = alg;
+    }
+    return alg;
+};
+
 
 /**
  * sign the given object with our ed25519 key