diff --git a/src/client.js b/src/client.js
index 860debc4b..c95583600 100644
--- a/src/client.js
+++ b/src/client.js
@@ -1723,7 +1723,7 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
         content: content,
     });
     localEvent._txnId = txnId;
-    localEvent.status = EventStatus.SENDING;
+    localEvent.setStatus(EventStatus.SENDING);
 
     // add this event immediately to the local store as 'sending'.
     if (room) {
@@ -1853,7 +1853,7 @@ function _updatePendingEventStatus(room, event, newStatus) {
     if (room) {
         room.updatePendingEvent(event, newStatus);
     } else {
-        event.status = newStatus;
+        event.setStatus(newStatus);
     }
 }
 
diff --git a/src/models/event.js b/src/models/event.js
index 753dc2b63..f01614061 100644
--- a/src/models/event.js
+++ b/src/models/event.js
@@ -731,7 +731,27 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     handleRemoteEcho: function(event) {
         this.event = event;
         // successfully sent.
-        this.status = null;
+        this.setStatus(null);
+    },
+
+    /**
+     * Whether the event is in any phase of sending, send failure, waiting for
+     * remote echo, etc.
+     *
+     * @return {boolean}
+     */
+    isSending() {
+        return !!this.status;
+    },
+
+    /**
+     * Update the event's sending status and emit an event as well.
+     *
+     * @param {String} status The new status
+     */
+    setStatus(status) {
+        this.status = status;
+        this.emit("Event.status", this, status);
     },
 
     /**
diff --git a/src/models/relations.js b/src/models/relations.js
index a74e65bf8..5d04d287f 100644
--- a/src/models/relations.js
+++ b/src/models/relations.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import EventEmitter from 'events';
+import { EventStatus } from '../../lib/models/event';
 
 /**
  * A container for relation events that supports easy access to common ways of
@@ -49,9 +50,55 @@ export default class Relations extends EventEmitter {
      * Add relation events to this collection.
      *
      * @param {MatrixEvent} event
-     * The new relation event to be aggregated.
+     * The new relation event to be added.
      */
     addEvent(event) {
+        if (this._relations.has(event)) {
+            return;
+        }
+
+        const relation = event.getRelation();
+        if (!relation) {
+            console.error("Event must have relation info");
+            return;
+        }
+
+        const relationType = relation.rel_type;
+        const eventType = event.getType();
+
+        if (this.relationType !== relationType || this.eventType !== eventType) {
+            console.error("Event relation info doesn't match this container");
+            return;
+        }
+
+        // If the event is in the process of being sent, listen for cancellation
+        // so we can remove the event from the collection.
+        if (event.isSending()) {
+            event.on("Event.status", this._onEventStatus);
+        }
+
+        if (this.relationType === "m.annotation") {
+            this._addAnnotationToAggregation(event);
+        }
+
+        this._relations.add(event);
+
+        event.on("Event.beforeRedaction", this._onBeforeRedaction);
+
+        this.emit("Relations.add", event);
+    }
+
+    /**
+     * Remove relation event from this collection.
+     *
+     * @param {MatrixEvent} event
+     * The relation event to remove.
+     */
+    _removeEvent(event) {
+        if (!this._relations.has(event)) {
+            return;
+        }
+
         const relation = event.getRelation();
         if (!relation) {
             console.error("Event must have relation info");
@@ -67,15 +114,32 @@ export default class Relations extends EventEmitter {
         }
 
         if (this.relationType === "m.annotation") {
-            const key = relation.key;
-            this._aggregateAnnotation(key, event);
+            this._removeAnnotationFromAggregation(event);
         }
 
-        this._relations.add(event);
+        this._relations.delete(event);
 
-        event.on("Event.beforeRedaction", this._onBeforeRedaction);
+        this.emit("Relations.remove", event);
+    }
 
-        this.emit("Relations.add", event);
+    /**
+     * Listens for event status changes to remove cancelled events.
+     *
+     * @param {MatrixEvent} event The event whose status has changed
+     * @param {EventStatus} status The new status
+     */
+    _onEventStatus = (event, status) => {
+        if (!event.isSending()) {
+            // Sending is done, so we don't need to listen anymore
+            event.removeListener("Event.status", this._onEventStatus);
+            return;
+        }
+        if (status !== EventStatus.CANCELLED) {
+            return;
+        }
+        // Event was cancelled, remove from the collection
+        event.removeListener("Event.status", this._onEventStatus);
+        this._removeEvent(event);
     }
 
     /**
@@ -92,7 +156,8 @@ export default class Relations extends EventEmitter {
         return [...this._relations];
     }
 
-    _aggregateAnnotation(key, event) {
+    _addAnnotationToAggregation(event) {
+        const { key } = event.getRelation();
         if (!key) {
             return;
         }
@@ -120,6 +185,28 @@ export default class Relations extends EventEmitter {
         eventsFromSender.push(event);
     }
 
+    _removeAnnotationFromAggregation(event) {
+        const { key } = event.getRelation();
+        if (!key) {
+            return;
+        }
+
+        const eventsForKey = this._annotationsByKey[key];
+        if (!eventsForKey) {
+            return;
+        }
+        eventsForKey.delete(event);
+
+        // Re-sort the [key, events] pairs in descending order of event count
+        this._sortedAnnotationsByKey.sort((a, b) => {
+            const aEvents = a[1];
+            const bEvents = b[1];
+            return bEvents.size - aEvents.size;
+        });
+
+        // TODO: Remove from events by sender if needed
+    }
+
     /**
      * For relations that have been redacted, we want to remove them from
      * aggregation data sets and emit an update event.
@@ -138,24 +225,7 @@ export default class Relations extends EventEmitter {
 
         if (this.relationType === "m.annotation") {
             // Remove the redacted annotation from aggregation by key
-            const relation = redactedEvent.getRelation();
-            if (!relation) {
-                return;
-            }
-
-            const key = relation.key;
-            const eventsForKey = this._annotationsByKey[key];
-            if (!eventsForKey) {
-                return;
-            }
-            eventsForKey.delete(redactedEvent);
-
-            // Re-sort the [key, events] pairs in descending order of event count
-            this._sortedAnnotationsByKey.sort((a, b) => {
-                const aEvents = a[1];
-                const bEvents = b[1];
-                return bEvents.size - aEvents.size;
-            });
+            this._removeAnnotationFromAggregation(redactedEvent);
         }
 
         redactedEvent.removeListener("Event.beforeRedaction", this._onBeforeRedaction);
diff --git a/src/models/room.js b/src/models/room.js
index b0a67053e..f8d7bdc69 100644
--- a/src/models/room.js
+++ b/src/models/room.js
@@ -1130,7 +1130,7 @@ Room.prototype.addPendingEvent = function(event, txnId) {
     if (this._opts.pendingEventOrdering == "detached") {
         if (this._pendingEventList.some((e) => e.status === EventStatus.NOT_SENT)) {
             console.warn("Setting event as NOT_SENT due to messages in the same state");
-            event.status = EventStatus.NOT_SENT;
+            event.setStatus(EventStatus.NOT_SENT);
         }
         this._pendingEventList.push(event);
 
@@ -1287,7 +1287,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
                         newStatus);
     }
 
-    event.status = newStatus;
+    event.setStatus(newStatus);
 
     if (newStatus == EventStatus.SENT) {
         // update the event id