From 546047a0508622742f6d613ab5803cd4287689bc Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Wed, 30 Oct 2024 11:52:34 -0400 Subject: [PATCH] Capture HTTP error response headers & handle Retry-After header (MSC4041) (#4471) * Include HTTP response headers in MatrixError * Lint * Support MSC4041 / Retry-After header * Fix tests * Remove redundant MatrixError parameter properties They are inherited from HTTPError, so there is no need to mark them as parameter properties. * Comment that retry_after_ms is deprecated * Properly handle colons in XHR header values Also remove the negation in the if-condition for better readability * Improve Retry-After parsing and docstring * Revert ternary operator to if statements for readability * Reuse resolved Headers for Content-Type parsing * Treat empty Content-Type differently from null * Add MatrixError#isRateLimitError This is separate from MatrixError#getRetryAfterMs because it's possible for a rate-limit error to have no Retry-After time, and having separate methods to check each makes that more clear. * Ignore HTTP status code when getting Retry-After because status codes other than 429 may have Retry-After * Catch Retry-After parsing errors * Add test coverage for HTTP error headers * Update license years * Move safe Retry-After lookup to global function so it can more conveniently check if an error is a MatrixError * Lint * Inline Retry-After header value parsing as it is only used in one place and doesn't need to be exported * Update docstrings Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> * Use bare catch Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> * Give HTTPError methods for rate-limit checks and make MatrixError inherit them * Cover undefined errcode in rate-limit check * Update safeGetRetryAfterMs docstring Be explicit that errors that don't look like rate-limiting errors will not pull a retry delay value from the error. * Use rate-limit helper functions in more places * Group the header tests --------- Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- spec/unit/http-api/errors.spec.ts | 98 ++++++++++++ spec/unit/http-api/index.spec.ts | 8 +- spec/unit/http-api/utils.spec.ts | 142 +++++++++++++----- src/http-api/errors.ts | 85 ++++++++++- src/http-api/utils.ts | 49 +++--- src/matrixrtc/MatrixRTCSession.ts | 6 +- .../PerSessionKeyBackupDownloader.ts | 20 +-- src/rust-crypto/backup.ts | 19 ++- 8 files changed, 345 insertions(+), 82 deletions(-) create mode 100644 spec/unit/http-api/errors.spec.ts diff --git a/spec/unit/http-api/errors.spec.ts b/spec/unit/http-api/errors.spec.ts new file mode 100644 index 000000000..426d23ee0 --- /dev/null +++ b/spec/unit/http-api/errors.spec.ts @@ -0,0 +1,98 @@ +/* +Copyright 2024 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixError } from "../../../src"; + +type IErrorJson = MatrixError["data"]; + +describe("MatrixError", () => { + let headers: Headers; + + beforeEach(() => { + headers = new Headers({ "Content-Type": "application/json" }); + }); + + function makeMatrixError(httpStatus: number, data: IErrorJson): MatrixError { + return new MatrixError(data, httpStatus, undefined, undefined, headers); + } + + it("should accept absent retry time from rate-limit error", () => { + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED" }); + expect(err.isRateLimitError()).toBe(true); + expect(err.getRetryAfterMs()).toEqual(null); + }); + + it("should retrieve retry_after_ms from rate-limit error", () => { + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 150000 }); + expect(err.isRateLimitError()).toBe(true); + expect(err.getRetryAfterMs()).toEqual(150000); + }); + + it("should ignore retry_after_ms if errcode is not M_LIMIT_EXCEEDED", () => { + const err = makeMatrixError(429, { errcode: "M_UNKNOWN", retry_after_ms: 150000 }); + expect(err.isRateLimitError()).toBe(true); + expect(err.getRetryAfterMs()).toEqual(null); + }); + + it("should retrieve numeric Retry-After header from rate-limit error", () => { + headers.set("Retry-After", "120"); + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 150000 }); + expect(err.isRateLimitError()).toBe(true); + // prefer Retry-After header over retry_after_ms + expect(err.getRetryAfterMs()).toEqual(120000); + }); + + it("should retrieve Date Retry-After header from rate-limit error", () => { + headers.set("Retry-After", `${new Date(160000).toUTCString()}`); + jest.spyOn(global.Date, "now").mockImplementationOnce(() => 100000); + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 150000 }); + expect(err.isRateLimitError()).toBe(true); + // prefer Retry-After header over retry_after_ms + expect(err.getRetryAfterMs()).toEqual(60000); + }); + + it("should prefer M_FORBIDDEN errcode over HTTP status code 429", () => { + headers.set("Retry-After", "120"); + const err = makeMatrixError(429, { errcode: "M_FORBIDDEN" }); + expect(err.isRateLimitError()).toBe(false); + // retrieve Retry-After header even for non-M_LIMIT_EXCEEDED errors + expect(err.getRetryAfterMs()).toEqual(120000); + }); + + it("should prefer M_LIMIT_EXCEEDED errcode over HTTP status code 400", () => { + headers.set("Retry-After", "120"); + const err = makeMatrixError(400, { errcode: "M_LIMIT_EXCEEDED" }); + expect(err.isRateLimitError()).toBe(true); + // retrieve Retry-After header even for non-429 errors + expect(err.getRetryAfterMs()).toEqual(120000); + }); + + it("should reject invalid Retry-After header", () => { + for (const invalidValue of ["-1", "1.23", new Date(0).toString()]) { + headers.set("Retry-After", invalidValue); + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED" }); + expect(() => err.getRetryAfterMs()).toThrow( + "value is not a valid HTTP-date or non-negative decimal integer", + ); + } + }); + + it("should reject too-large Retry-After header", () => { + headers.set("Retry-After", "1" + Array(500).fill("0").join("")); + const err = makeMatrixError(429, { errcode: "M_LIMIT_EXCEEDED" }); + expect(() => err.getRetryAfterMs()).toThrow("integer value is too large"); + }); +}); diff --git a/spec/unit/http-api/index.spec.ts b/spec/unit/http-api/index.spec.ts index 668417ea9..ca20bee45 100644 --- a/spec/unit/http-api/index.spec.ts +++ b/spec/unit/http-api/index.spec.ts @@ -1,5 +1,5 @@ /* -Copyright 2022 The Matrix.org Foundation C.I.C. +Copyright 2022 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ describe("MatrixHttpApi", () => { setRequestHeader: jest.fn(), onreadystatechange: undefined, getResponseHeader: jest.fn(), + getAllResponseHeaders: jest.fn(), } as unknown as XMLHttpRequest; // We stub out XHR here as it is not available in JSDOM // @ts-ignore @@ -171,7 +172,10 @@ describe("MatrixHttpApi", () => { xhr.readyState = DONE; xhr.responseText = '{"errcode": "M_NOT_FOUND", "error": "Not found"}'; xhr.status = 404; - mocked(xhr.getResponseHeader).mockReturnValue("application/json"); + mocked(xhr.getResponseHeader).mockImplementation((name) => + name.toLowerCase() === "content-type" ? "application/json" : null, + ); + mocked(xhr.getAllResponseHeaders).mockReturnValue("content-type: application/json\r\n"); // @ts-ignore xhr.onreadystatechange?.(new Event("test")); diff --git a/spec/unit/http-api/utils.spec.ts b/spec/unit/http-api/utils.spec.ts index 9d74f79dc..92c1e8ac2 100644 --- a/spec/unit/http-api/utils.spec.ts +++ b/spec/unit/http-api/utils.spec.ts @@ -1,5 +1,5 @@ /* -Copyright 2022 The Matrix.org Foundation C.I.C. +Copyright 2022 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -86,13 +86,28 @@ describe("anySignal", () => { }); describe("parseErrorResponse", () => { + let headers: Headers; + const xhrHeaderMethods = { + getResponseHeader: (name: string) => headers.get(name), + getAllResponseHeaders: () => { + let allHeaders = ""; + headers.forEach((value, key) => { + allHeaders += `${key.toLowerCase()}: ${value}\r\n`; + }); + return allHeaders; + }, + }; + + beforeEach(() => { + headers = new Headers(); + }); + it("should resolve Matrix Errors from XHR", () => { + headers.set("Content-Type", "application/json"); expect( parseErrorResponse( { - getResponseHeader(name: string): string | null { - return name === "Content-Type" ? "application/json" : null; - }, + ...xhrHeaderMethods, status: 500, } as XMLHttpRequest, '{"errcode": "TEST"}', @@ -108,14 +123,11 @@ describe("parseErrorResponse", () => { }); it("should resolve Matrix Errors from fetch", () => { + headers.set("Content-Type", "application/json"); expect( parseErrorResponse( { - headers: { - get(name: string): string | null { - return name === "Content-Type" ? "application/json" : null; - }, - }, + headers, status: 500, } as Response, '{"errcode": "TEST"}', @@ -131,13 +143,12 @@ describe("parseErrorResponse", () => { }); it("should resolve Matrix Errors from XHR with urls", () => { + headers.set("Content-Type", "application/json"); expect( parseErrorResponse( { responseURL: "https://example.com", - getResponseHeader(name: string): string | null { - return name === "Content-Type" ? "application/json" : null; - }, + ...xhrHeaderMethods, status: 500, } as XMLHttpRequest, '{"errcode": "TEST"}', @@ -154,15 +165,12 @@ describe("parseErrorResponse", () => { }); it("should resolve Matrix Errors from fetch with urls", () => { + headers.set("Content-Type", "application/json"); expect( parseErrorResponse( { url: "https://example.com", - headers: { - get(name: string): string | null { - return name === "Content-Type" ? "application/json" : null; - }, - }, + headers, status: 500, } as Response, '{"errcode": "TEST"}', @@ -178,6 +186,66 @@ describe("parseErrorResponse", () => { ); }); + describe("with HTTP headers", () => { + function addHeaders(headers: Headers) { + headers.set("Age", "0"); + headers.set("Date", "Thu, 01 Jan 1970 00:00:00 GMT"); // value contains colons + headers.set("x-empty", ""); + headers.set("x-multi", "1"); + headers.append("x-multi", "2"); + } + + function compareHeaders(expectedHeaders: Headers, otherHeaders: Headers | undefined) { + expect(new Map(otherHeaders)).toEqual(new Map(expectedHeaders)); + } + + it("should resolve HTTP Errors from XHR with headers", () => { + headers.set("Content-Type", "text/plain"); + addHeaders(headers); + const err = parseErrorResponse({ + ...xhrHeaderMethods, + status: 500, + } as XMLHttpRequest) as HTTPError; + compareHeaders(headers, err.httpHeaders); + }); + + it("should resolve HTTP Errors from fetch with headers", () => { + headers.set("Content-Type", "text/plain"); + addHeaders(headers); + const err = parseErrorResponse({ + headers, + status: 500, + } as Response) as HTTPError; + compareHeaders(headers, err.httpHeaders); + }); + + it("should resolve Matrix Errors from XHR with headers", () => { + headers.set("Content-Type", "application/json"); + addHeaders(headers); + const err = parseErrorResponse( + { + ...xhrHeaderMethods, + status: 500, + } as XMLHttpRequest, + '{"errcode": "TEST"}', + ) as MatrixError; + compareHeaders(headers, err.httpHeaders); + }); + + it("should resolve Matrix Errors from fetch with headers", () => { + headers.set("Content-Type", "application/json"); + addHeaders(headers); + const err = parseErrorResponse( + { + headers, + status: 500, + } as Response, + '{"errcode": "TEST"}', + ) as MatrixError; + compareHeaders(headers, err.httpHeaders); + }); + }); + it("should set a sensible default error message on MatrixError", () => { let err = new MatrixError(); expect(err.message).toEqual("MatrixError: Unknown message"); @@ -188,14 +256,11 @@ describe("parseErrorResponse", () => { }); it("should handle no type gracefully", () => { + // No Content-Type header expect( parseErrorResponse( { - headers: { - get(name: string): string | null { - return null; - }, - }, + headers, status: 500, } as Response, '{"errcode": "TEST"}', @@ -203,31 +268,38 @@ describe("parseErrorResponse", () => { ).toStrictEqual(new HTTPError("Server returned 500 error", 500)); }); - it("should handle invalid type gracefully", () => { + it("should handle empty type gracefully", () => { + headers.set("Content-Type", " "); expect( parseErrorResponse( { - headers: { - get(name: string): string | null { - return name === "Content-Type" ? " " : null; - }, - }, + headers, status: 500, } as Response, '{"errcode": "TEST"}', ), - ).toStrictEqual(new Error("Error parsing Content-Type ' ': TypeError: invalid media type")); + ).toStrictEqual(new Error("Error parsing Content-Type '': TypeError: argument string is required")); }); - it("should handle plaintext errors", () => { + it("should handle invalid type gracefully", () => { + headers.set("Content-Type", "unknown"); expect( parseErrorResponse( { - headers: { - get(name: string): string | null { - return name === "Content-Type" ? "text/plain" : null; - }, - }, + headers, + status: 500, + } as Response, + '{"errcode": "TEST"}', + ), + ).toStrictEqual(new Error("Error parsing Content-Type 'unknown': TypeError: invalid media type")); + }); + + it("should handle plaintext errors", () => { + headers.set("Content-Type", "text/plain"); + expect( + parseErrorResponse( + { + headers, status: 418, } as Response, "I'm a teapot", diff --git a/src/http-api/errors.ts b/src/http-api/errors.ts index 86cfdc908..f80c3fdd8 100644 --- a/src/http-api/errors.ts +++ b/src/http-api/errors.ts @@ -1,5 +1,5 @@ /* -Copyright 2022 The Matrix.org Foundation C.I.C. +Copyright 2022 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,14 +28,53 @@ interface IErrorJson extends Partial { * specific to HTTP responses. * @param msg - The error message to include. * @param httpStatus - The HTTP response status code. + * @param httpHeaders - The HTTP response headers. */ export class HTTPError extends Error { public constructor( msg: string, public readonly httpStatus?: number, + public readonly httpHeaders?: Headers, ) { super(msg); } + + /** + * Check if this error was due to rate-limiting on the server side (and should therefore be retried after a delay). + * + * If this returns `true`, {@link getRetryAfterMs} can be called to retrieve the server-side + * recommendation for the retry period. + * + * @returns Whether this error is due to rate-limiting. + */ + public isRateLimitError(): boolean { + return this.httpStatus === 429; + } + + /** + * @returns The recommended delay in milliseconds to wait before retrying + * the request that triggered this error, or null if no delay is recommended. + * @throws Error if the recommended delay is an invalid value. + * @see {@link safeGetRetryAfterMs} for a version of this check that doesn't throw. + */ + public getRetryAfterMs(): number | null { + const retryAfter = this.httpHeaders?.get("Retry-After"); + if (retryAfter != null) { + if (/^\d+$/.test(retryAfter)) { + const ms = Number.parseInt(retryAfter) * 1000; + if (!Number.isFinite(ms)) { + throw new Error("Retry-After header integer value is too large"); + } + return ms; + } + const date = new Date(retryAfter); + if (date.toUTCString() !== retryAfter) { + throw new Error("Retry-After header value is not a valid HTTP-date or non-negative decimal integer"); + } + return date.getTime() - Date.now(); + } + return null; + } } export class MatrixError extends HTTPError { @@ -49,12 +88,14 @@ export class MatrixError extends HTTPError { * information specific to the standard Matrix error response. * @param errorJson - The Matrix error JSON returned from the homeserver. * @param httpStatus - The numeric HTTP status code given + * @param httpHeaders - The HTTP response headers given */ public constructor( errorJson: IErrorJson = {}, - public readonly httpStatus?: number, + httpStatus?: number, public url?: string, public event?: MatrixEvent, + httpHeaders?: Headers, ) { let message = errorJson.error || "Unknown message"; if (httpStatus) { @@ -63,11 +104,49 @@ export class MatrixError extends HTTPError { if (url) { message = `${message} (${url})`; } - super(`MatrixError: ${message}`, httpStatus); + super(`MatrixError: ${message}`, httpStatus, httpHeaders); this.errcode = errorJson.errcode; this.name = errorJson.errcode || "Unknown error code"; this.data = errorJson; } + + public isRateLimitError(): boolean { + return ( + this.errcode === "M_LIMIT_EXCEEDED" || + ((this.errcode === "M_UNKNOWN" || this.errcode === undefined) && super.isRateLimitError()) + ); + } + + public getRetryAfterMs(): number | null { + const headerValue = super.getRetryAfterMs(); + if (headerValue !== null) { + return headerValue; + } + // Note: retry_after_ms is deprecated as of spec version v1.10 + if (this.errcode === "M_LIMIT_EXCEEDED" && "retry_after_ms" in this.data) { + if (!Number.isInteger(this.data.retry_after_ms)) { + throw new Error("retry_after_ms is not an integer"); + } + return this.data.retry_after_ms; + } + return null; + } +} + +/** + * @returns The recommended delay in milliseconds to wait before retrying + * the request that triggered {@link error}, or {@link defaultMs} if the + * error was not due to rate-limiting or if no valid delay is recommended. + */ +export function safeGetRetryAfterMs(error: unknown, defaultMs: number): number { + if (!(error instanceof HTTPError) || !error.isRateLimitError()) { + return defaultMs; + } + try { + return error.getRetryAfterMs() ?? defaultMs; + } catch { + return defaultMs; + } } /** diff --git a/src/http-api/utils.ts b/src/http-api/utils.ts index 861d47ea1..d23e840bc 100644 --- a/src/http-api/utils.ts +++ b/src/http-api/utils.ts @@ -1,5 +1,5 @@ /* -Copyright 2022 The Matrix.org Foundation C.I.C. +Copyright 2022 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import { parse as parseContentType, ParsedMediaType } from "content-type"; import { logger } from "../logger.ts"; import { sleep } from "../utils.ts"; -import { ConnectionError, HTTPError, MatrixError } from "./errors.ts"; +import { ConnectionError, HTTPError, MatrixError, safeGetRetryAfterMs } from "./errors.ts"; // Ponyfill for https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout export function timeoutSignal(ms: number): AbortSignal { @@ -72,24 +72,38 @@ export function anySignal(signals: AbortSignal[]): { * @returns */ export function parseErrorResponse(response: XMLHttpRequest | Response, body?: string): Error { + const httpHeaders = isXhr(response) + ? new Headers( + response + .getAllResponseHeaders() + .trim() + .split(/[\r\n]+/) + .map((header): [string, string] => { + const colonIdx = header.indexOf(":"); + return [header.substring(0, colonIdx), header.substring(colonIdx + 1)]; + }), + ) + : response.headers; + let contentType: ParsedMediaType | null; try { - contentType = getResponseContentType(response); + contentType = getResponseContentType(httpHeaders); } catch (e) { return e; } - if (contentType?.type === "application/json" && body) { return new MatrixError( JSON.parse(body), response.status, isXhr(response) ? response.responseURL : response.url, + undefined, + httpHeaders, ); } if (contentType?.type === "text/plain") { - return new HTTPError(`Server returned ${response.status} error: ${body}`, response.status); + return new HTTPError(`Server returned ${response.status} error: ${body}`, response.status, httpHeaders); } - return new HTTPError(`Server returned ${response.status} error`, response.status); + return new HTTPError(`Server returned ${response.status} error`, response.status, httpHeaders); } function isXhr(response: XMLHttpRequest | Response): response is XMLHttpRequest { @@ -97,7 +111,7 @@ function isXhr(response: XMLHttpRequest | Response): response is XMLHttpRequest } /** - * extract the Content-Type header from the response object, and + * extract the Content-Type header from response headers, and * parse it to a `{type, parameters}` object. * * returns null if no content-type header could be found. @@ -105,15 +119,9 @@ function isXhr(response: XMLHttpRequest | Response): response is XMLHttpRequest * @param response - response object * @returns parsed content-type header, or null if not found */ -function getResponseContentType(response: XMLHttpRequest | Response): ParsedMediaType | null { - let contentType: string | null; - if (isXhr(response)) { - contentType = response.getResponseHeader("Content-Type"); - } else { - contentType = response.headers.get("Content-Type"); - } - - if (!contentType) return null; +function getResponseContentType(headers: Headers): ParsedMediaType | null { + const contentType = headers.get("Content-Type"); + if (contentType === null) return null; try { return parseContentType(contentType); @@ -188,12 +196,5 @@ export function calculateRetryBackoff(err: any, attempts: number, retryConnectio return -1; } - if (err.name === "M_LIMIT_EXCEEDED") { - const waitTime = err.data.retry_after_ms; - if (waitTime > 0) { - return waitTime; - } - } - - return 1000 * Math.pow(2, attempts); + return safeGetRetryAfterMs(err, 1000 * Math.pow(2, attempts)); } diff --git a/src/matrixrtc/MatrixRTCSession.ts b/src/matrixrtc/MatrixRTCSession.ts index 0e4b385b9..52258ea53 100644 --- a/src/matrixrtc/MatrixRTCSession.ts +++ b/src/matrixrtc/MatrixRTCSession.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 The Matrix.org Foundation C.I.C. +Copyright 2023 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ import { randomString, secureRandomBase64Url } from "../randomstring.ts"; import { EncryptionKeysEventContent } from "./types.ts"; import { decodeBase64, encodeUnpaddedBase64 } from "../base64.ts"; import { KnownMembership } from "../@types/membership.ts"; -import { MatrixError } from "../http-api/errors.ts"; +import { MatrixError, safeGetRetryAfterMs } from "../http-api/errors.ts"; import { MatrixEvent } from "../models/event.ts"; import { isLivekitFocusActive } from "./LivekitFocus.ts"; import { ExperimentalGroupCallRoomMemberState } from "../webrtc/groupCall.ts"; @@ -630,7 +630,7 @@ export class MatrixRTCSession extends TypedEventEmitter 0) { - this.logger.info(`Rate limited by server, waiting ${waitTime}ms`); - throw new KeyDownloadRateLimitError(waitTime); - } else { - // apply the default backoff time - throw new KeyDownloadRateLimitError(KEY_BACKUP_BACKOFF); + if (e.isRateLimitError()) { + let waitTime: number | undefined; + try { + waitTime = e.getRetryAfterMs() ?? undefined; + } catch (error) { + this.logger.warn("Error while retrieving a rate-limit retry delay", error); } + if (waitTime && waitTime > 0) { + this.logger.info(`Rate limited by server, waiting ${waitTime}ms`); + } + throw new KeyDownloadRateLimitError(waitTime ?? KEY_BACKUP_BACKOFF); } } throw new KeyDownloadError(KeyDownloadErrorCode.NETWORK_ERROR); diff --git a/src/rust-crypto/backup.ts b/src/rust-crypto/backup.ts index 63d1afdb0..59aa30de3 100644 --- a/src/rust-crypto/backup.ts +++ b/src/rust-crypto/backup.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 The Matrix.org Foundation C.I.C. +Copyright 2023 - 2024 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -457,12 +457,19 @@ export class RustBackupManager extends TypedEventEmitter 0) { - await sleep(waitTime); - continue; + try { + const waitTime = err.getRetryAfterMs(); + if (waitTime && waitTime > 0) { + await sleep(waitTime); + continue; + } + } catch (error) { + logger.warn( + "Backup: An error occurred while retrieving a rate-limit retry delay", + error, + ); } // else go to the normal backoff } }