Custom abort timeout logic for restarting delayed events that is compatible with the widget api (#4927)

* add custom local timout + add delay to 0 for normal local timeout.

* consider retry limits for new custom error

* mock the AbortError so we can reuse `actionUpdateFromErrors`

* update comment

src/matrixrtc/MatrixRTCSession.ts

@@ -79,7 +79,7 @@ export interface SessionConfig {
 // - we use delayedLeaveEvent if the option is related to the delayed leave event.
 // - we use membershipEvent if the option is related to the rtc member state event.
 // - we use the technical term expiry if the option is related to the expiry field of the membership state event.
-// - we use a `MS` postfix if the option is a duration to avoid using words like:
+// - we use a `Ms` postfix if the option is a duration to avoid using words like:
 //   `time`, `duration`, `delay`, `timeout`... that might be mistaken/confused with technical terms.
 export interface MembershipConfig {
     /**
@@ -143,6 +143,7 @@ export interface MembershipConfig {
      * failed to send due to a network error. (send membership event, send delayed event, restart delayed event...)
      */
     networkErrorRetryMs?: number;
+
     /** @deprecated renamed to `networkErrorRetryMs`*/
     callMemberEventRetryDelayMinimum?: number;
 
@@ -150,6 +151,17 @@ export interface MembershipConfig {
      * If true, use the new to-device transport for sending encryption keys.
      */
     useExperimentalToDeviceTransport?: boolean;
+
+    /**
+     * The time (in milliseconds) after which a we consider a delayed event restart http request to have failed.
+     * Setting this to a lower value will result in more frequent retries but also a higher chance of failiour.
+     *
+     * In the presence of network packet loss (hurting TCP connections), the custom delayedEventRestartLocalTimeoutMs
+     * helps by keeping more delayed event reset candidates in flight,
+     * improving the chances of a successful reset. (its is equivalent to the js-sdk `localTimeout` configuration,
+     * but only applies to calls to the `_unstable_updateDelayedEvent` endpoint with a body of `{action:"restart"}`.)
+     */
+    delayedLeaveEventRestartLocalTimeoutMs?: number;
 }
 
 export interface EncryptionConfig {

src/matrixrtc/MembershipManager.ts

@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import { AbortError } from "p-retry";
 
 import { EventType } from "../@types/event.ts";
 import { UpdateDelayedEventAction } from "../@types/requests.ts";
@@ -85,19 +86,24 @@ export enum MembershipActionType {
     //  -> MembershipActionType.SendJoinEvent if successful
     //  -> DelayedLeaveActionType.SendDelayedEvent on error, retry sending the first delayed event.
     //  -> DelayedLeaveActionType.RestartDelayedEvent on success start updating the delayed event
+
     SendJoinEvent = "SendJoinEvent",
     //  -> MembershipActionType.SendJoinEvent if we run into a rate limit and need to retry
     //  -> MembershipActionType.Update if we successfully send the join event then schedule the expire event update
     //  -> DelayedLeaveActionType.RestartDelayedEvent to recheck the delayed event
+
     RestartDelayedEvent = "RestartDelayedEvent",
     //  -> DelayedLeaveActionType.SendMainDelayedEvent on missing delay id but there is a rtc state event
     //  -> DelayedLeaveActionType.SendDelayedEvent on missing delay id and there is no state event
     //  -> DelayedLeaveActionType.RestartDelayedEvent on success we schedule the next restart
+
     UpdateExpiry = "UpdateExpiry",
     //  -> MembershipActionType.Update if the timeout has passed so the next update is required.
+
     SendScheduledDelayedLeaveEvent = "SendScheduledDelayedLeaveEvent",
     //  -> MembershipActionType.SendLeaveEvent on failiour (not found) we need to send the leave manually and cannot use the scheduled delayed event
     //  -> DelayedLeaveActionType.SendScheduledDelayedLeaveEvent on error we try again.
+
     SendLeaveEvent = "SendLeaveEvent",
     // -> MembershipActionType.SendLeaveEvent
 }
@@ -385,6 +391,9 @@ export class MembershipManager
         return this.joinConfig?.maximumNetworkErrorRetryCount ?? 10;
     }
 
+    private get delayedLeaveEventRestartLocalTimeoutMs(): number {
+        return this.joinConfig?.delayedLeaveEventRestartLocalTimeoutMs ?? 2000;
+    }
     // LOOP HANDLER:
     private async membershipLoopHandler(type: MembershipActionType): Promise<ActionUpdate> {
         switch (type) {
@@ -536,8 +545,18 @@ export class MembershipManager
     }
 
     private async restartDelayedEvent(delayId: string): Promise<ActionUpdate> {
-        return await this.client
-            ._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart)
+        const abortPromise = new Promise((_, reject) => {
+            setTimeout(() => {
+                reject(new AbortError("Restart delayed event timed out before the HS responded"));
+            }, this.delayedLeaveEventRestartLocalTimeoutMs);
+        });
+
+        // The obvious choice here would be to use the `IRequestOpts` to set the timeout. Since this call might be forwarded
+        // to the widget driver this information would ge lost. That is why we mimic the AbortError using the race.
+        return await Promise.race([
+            this.client._unstable_updateDelayedEvent(delayId, UpdateDelayedEventAction.Restart),
+            abortPromise,
+        ])
             .then(() => {
                 this.resetRateLimitCounter(MembershipActionType.RestartDelayedEvent);
                 return createInsertActionUpdate(
@@ -786,14 +805,19 @@ export class MembershipManager
     private actionUpdateFromNetworkErrorRetry(error: unknown, type: MembershipActionType): ActionUpdate | undefined {
         // "Is a network error"-boundary
         const retries = this.state.networkErrorRetries.get(type) ?? 0;
+
+        // Strings for error logging
         const retryDurationString = this.networkErrorRetryMs / 1000 + "s";
         const retryCounterString = "(" + retries + "/" + this.maximumNetworkErrorRetryCount + ")";
+
+        // Variables for scheduling the new event
+        let retryDuration = this.networkErrorRetryMs;
+
         if (error instanceof Error && error.name === "AbortError") {
+            // We do not wait for the timeout on local timeouts.
+            retryDuration = 0;
             this.logger.warn(
-                "Network local timeout error while sending event, retrying in " +
-                    retryDurationString +
-                    " " +
-                    retryCounterString,
+                "Network local timeout error while sending event, immediate retry (" + retryCounterString + ")",
                 error,
             );
         } else if (error instanceof Error && error.message.includes("updating delayed event")) {
@@ -836,7 +860,7 @@ export class MembershipManager
         // retry boundary
         if (retries < this.maximumNetworkErrorRetryCount) {
             this.state.networkErrorRetries.set(type, retries + 1);
-            return createInsertActionUpdate(type, this.networkErrorRetryMs);
+            return createInsertActionUpdate(type, retryDuration);
         }
 
         // Failure