Populate info.duration for audio & video file uploads (#11225)

* Improve m.file m.image m.audio m.video types * Populate `info.duration` for audio & video file uploads * Fix tests * Iterate types * Improve coverage * Fix test * Add small delay to stabilise cypress test * Fix test idempotency * Improve coverage * Slow down * iterate
2025-08-07 21:23:00 +03:00 · 2023-07-17 13:07:58 +01:00
parent 8b8ca425d7
commit f04a0e2860
17 changed files with 556 additions and 85 deletions
--- a/test/ContentMessages-test.ts
+++ b/test/ContentMessages-test.ts
@@ -163,6 +163,11 @@ describe("ContentMessages", () => {
                            return 800;
                        },
                    });
+                    Object.defineProperty(element, "duration", {
+                        get() {
+                            return 123;
+                        },
+                    });
                }
                return element;
            });
@@ -176,11 +181,31 @@ describe("ContentMessages", () => {
                expect.objectContaining({
                    url: "mxc://server/file",
                    msgtype: "m.video",
+                    info: expect.objectContaining({
+                        duration: 123000,
+                    }),
                }),
            );
        });

        it("should use m.audio for audio files", async () => {
+            jest.spyOn(document, "createElement").mockImplementation((tagName) => {
+                const element = createElement(tagName);
+                if (tagName === "audio") {
+                    Object.defineProperty(element, "duration", {
+                        get() {
+                            return 621;
+                        },
+                    });
+                    Object.defineProperty(element, "src", {
+                        set() {
+                            element.onloadedmetadata!(new Event("loadedmetadata"));
+                        },
+                    });
+                }
+                return element;
+            });
+
            mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
            const file = new File([], "fileName", { type: "audio/mp3" });
            await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
@@ -190,6 +215,34 @@ describe("ContentMessages", () => {
                expect.objectContaining({
                    url: "mxc://server/file",
                    msgtype: "m.audio",
+                    info: expect.objectContaining({
+                        duration: 621000,
+                    }),
+                }),
+            );
+        });
+
+        it("should fall back to m.file for invalid audio files", async () => {
+            jest.spyOn(document, "createElement").mockImplementation((tagName) => {
+                const element = createElement(tagName);
+                if (tagName === "audio") {
+                    Object.defineProperty(element, "src", {
+                        set() {
+                            element.onerror!("fail");
+                        },
+                    });
+                }
+                return element;
+            });
+            mocked(client.uploadContent).mockResolvedValue({ content_uri: "mxc://server/file" });
+            const file = new File([], "fileName", { type: "audio/mp3" });
+            await contentMessages.sendContentToRoom(file, roomId, undefined, client, undefined);
+            expect(client.sendMessage).toHaveBeenCalledWith(
+                roomId,
+                null,
+                expect.objectContaining({
+                    url: "mxc://server/file",
+                    msgtype: "m.file",
                }),
            );
        });
--- a/test/components/views/messages/MImageBody-test.tsx
+++ b/test/components/views/messages/MImageBody-test.tsx
@@ -15,11 +15,13 @@ limitations under the License.
 */

 import React from "react";
-import { fireEvent, render, screen } from "@testing-library/react";
+import { fireEvent, render, screen, waitForElementToBeRemoved } from "@testing-library/react";
 import { EventType, getHttpUriForMxc, MatrixEvent, Room } from "matrix-js-sdk/src/matrix";
 import fetchMock from "fetch-mock-jest";
 import encrypt from "matrix-encrypt-attachment";
 import { mocked } from "jest-mock";
+import fs from "fs";
+import path from "path";

 import MImageBody from "../../../../src/components/views/messages/MImageBody";
 import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
@@ -56,7 +58,11 @@ describe("<MImageBody/>", () => {
    });
    const url = "https://server/_matrix/media/r0/download/server/encrypted-image";
    // eslint-disable-next-line no-restricted-properties
-    cli.mxcUrlToHttp.mockReturnValue(url);
+    cli.mxcUrlToHttp.mockImplementation(
+        (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
+            return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
+        },
+    );
    const encryptedMediaEvent = new MatrixEvent({
        room_id: "!room:server",
        sender: userId,
@@ -175,12 +181,6 @@ describe("<MImageBody/>", () => {
    it("should fall back to /download/ if /thumbnail/ fails", async () => {
        const thumbUrl = "https://server/_matrix/media/r0/thumbnail/server/image?width=800&height=600&method=scale";
        const downloadUrl = "https://server/_matrix/media/r0/download/server/image";
-        // eslint-disable-next-line no-restricted-properties
-        cli.mxcUrlToHttp.mockImplementation(
-            (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
-                return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
-            },
-        );

        const event = new MatrixEvent({
            room_id: "!room:server",
@@ -206,4 +206,56 @@ describe("<MImageBody/>", () => {
        fireEvent.error(img);
        expect(img).toHaveProperty("src", downloadUrl);
    });
+
+    it("should generate a thumbnail if one isn't included for animated media", async () => {
+        Object.defineProperty(global.Image.prototype, "src", {
+            set(src) {
+                window.setTimeout(() => this.onload());
+            },
+        });
+        Object.defineProperty(global.Image.prototype, "height", {
+            get() {
+                return 600;
+            },
+        });
+        Object.defineProperty(global.Image.prototype, "width", {
+            get() {
+                return 800;
+            },
+        });
+
+        mocked(global.URL.createObjectURL).mockReturnValue("blob:generated-thumb");
+
+        fetchMock.getOnce(
+            "https://server/_matrix/media/r0/download/server/image",
+            {
+                body: fs.readFileSync(path.resolve(__dirname, "..", "..", "..", "images", "animated-logo.webp")),
+            },
+            { sendAsJson: false },
+        );
+
+        const event = new MatrixEvent({
+            room_id: "!room:server",
+            sender: userId,
+            type: EventType.RoomMessage,
+            content: {
+                body: "alt for a test image",
+                info: {
+                    w: 40,
+                    h: 50,
+                    mimetype: "image/webp",
+                },
+                url: "mxc://server/image",
+            },
+        });
+
+        const { container } = render(
+            <MImageBody {...props} mxEvent={event} mediaEventHelper={new MediaEventHelper(event)} />,
+        );
+
+        // Wait for spinners to go away
+        await waitForElementToBeRemoved(screen.getAllByRole("progressbar"));
+        // thumbnail with dimensions present
+        expect(container).toMatchSnapshot();
+    });
 });
--- a/test/components/views/messages/MVideoBody-test.tsx
+++ b/test/components/views/messages/MVideoBody-test.tsx
@@ -15,23 +15,22 @@ limitations under the License.
 */

 import React, { ComponentProps } from "react";
-import { IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
+import { EventType, getHttpUriForMxc, IContent, MatrixEvent } from "matrix-js-sdk/src/matrix";
 import { render, RenderResult } from "@testing-library/react";
+import fetchMock from "fetch-mock-jest";

 import MatrixClientContext from "../../../../src/contexts/MatrixClientContext";
 import { RoomPermalinkCreator } from "../../../../src/utils/permalinks/Permalinks";
 import { MediaEventHelper } from "../../../../src/utils/MediaEventHelper";
-import { getMockClientWithEventEmitter } from "../../../test-utils";
+import {
+    getMockClientWithEventEmitter,
+    mockClientMethodsCrypto,
+    mockClientMethodsDevice,
+    mockClientMethodsServer,
+    mockClientMethodsUser,
+} from "../../../test-utils";
 import MVideoBody from "../../../../src/components/views/messages/MVideoBody";

-jest.mock("../../../../src/customisations/Media", () => {
-    return {
-        mediaFromContent: () => {
-            return { isEncrypted: false };
-        },
-    };
-});
-
 describe("MVideoBody", () => {
    it("does not crash when given a portrait image", () => {
        // Check for an unreliable crash caused by a fractional-sized
@@ -40,6 +39,58 @@ describe("MVideoBody", () => {
        expect(asFragment()).toMatchSnapshot();
        // If we get here, we did not crash.
    });
+
+    it("should show poster for encrypted media before downloading it", async () => {
+        const userId = "@user:server";
+        const deviceId = "DEADB33F";
+        const cli = getMockClientWithEventEmitter({
+            ...mockClientMethodsUser(userId),
+            ...mockClientMethodsServer(),
+            ...mockClientMethodsDevice(deviceId),
+            ...mockClientMethodsCrypto(),
+            getRooms: jest.fn().mockReturnValue([]),
+            getIgnoredUsers: jest.fn(),
+            getVersions: jest.fn().mockResolvedValue({
+                unstable_features: {
+                    "org.matrix.msc3882": true,
+                    "org.matrix.msc3886": true,
+                },
+            }),
+        });
+        const thumbUrl = "https://server/_matrix/media/r0/download/server/encrypted-poster";
+        fetchMock.getOnce(thumbUrl, { status: 200 });
+
+        // eslint-disable-next-line no-restricted-properties
+        cli.mxcUrlToHttp.mockImplementation(
+            (mxcUrl: string, width?: number, height?: number, resizeMethod?: string, allowDirectLinks?: boolean) => {
+                return getHttpUriForMxc("https://server", mxcUrl, width, height, resizeMethod, allowDirectLinks);
+            },
+        );
+        const encryptedMediaEvent = new MatrixEvent({
+            room_id: "!room:server",
+            sender: userId,
+            type: EventType.RoomMessage,
+            content: {
+                body: "alt for a test video",
+                info: {
+                    duration: 420,
+                    w: 40,
+                    h: 50,
+                    thumbnail_file: {
+                        url: "mxc://server/encrypted-poster",
+                    },
+                },
+                file: {
+                    url: "mxc://server/encrypted-image",
+                },
+            },
+        });
+
+        const { asFragment } = render(
+            <MVideoBody mxEvent={encryptedMediaEvent} mediaEventHelper={new MediaEventHelper(encryptedMediaEvent)} />,
+        );
+        expect(asFragment()).toMatchSnapshot();
+    });
 });

 function makeMVideoBody(w: number, h: number): RenderResult {
--- a/test/components/views/messages/snapshots/MImageBody-test.tsx.snap
+++ b/test/components/views/messages/snapshots/MImageBody-test.tsx.snap
@@ -1,5 +1,55 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP

+exports[`<MImageBody/> should generate a thumbnail if one isn't included for animated media 1`] = `
+<div>
+  <div
+    class="mx_MImageBody"
+  >
+    <a
+      href="https://server/_matrix/media/r0/download/server/image"
+    >
+      <div
+        class="mx_MImageBody_thumbnail_container"
+        style="max-height: 50px; max-width: 40px;"
+      >
+        <div
+          class="mx_MImageBody_placeholder"
+        >
+          <div
+            class="mx_Spinner"
+          >
+            <div
+              aria-label="Loading…"
+              class="mx_Spinner_icon"
+              data-testid="spinner"
+              role="progressbar"
+              style="width: 32px; height: 32px;"
+            />
+          </div>
+        </div>
+        <div
+          style="max-height: 50px; max-width: 40px;"
+        >
+          <img
+            alt="alt for a test image"
+            class="mx_MImageBody_thumbnail"
+            src="blob:generated-thumb"
+          />
+          <p
+            class="mx_MImageBody_gifLabel"
+          >
+            GIF
+          </p>
+        </div>
+        <div
+          style="height: 50px; width: 40px;"
+        />
+      </div>
+    </a>
+  </div>
+</div>
+`;
+
 exports[`<MImageBody/> should show a thumbnail while image is being downloaded 1`] = `
 <div>
  <div
--- a/test/components/views/messages/snapshots/MVideoBody-test.tsx.snap
+++ b/test/components/views/messages/snapshots/MVideoBody-test.tsx.snap
@@ -23,3 +23,28 @@ exports[`MVideoBody does not crash when given a portrait image 1`] = `
  </span>
 </DocumentFragment>
 `;
+
+exports[`MVideoBody should show poster for encrypted media before downloading it 1`] = `
+<DocumentFragment>
+  <span
+    class="mx_MVideoBody"
+  >
+    <div
+      class="mx_MVideoBody_container"
+      style="max-width: 40px; max-height: 50px;"
+    >
+      <video
+        class="mx_MVideoBody"
+        controls=""
+        controlslist="nodownload"
+        poster="https://server/_matrix/media/r0/download/server/encrypted-poster"
+        preload="none"
+        title="alt for a test video"
+      />
+      <div
+        style="width: 40px; height: 50px;"
+      />
+    </div>
+  </span>
+</DocumentFragment>
+`;
--- a/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts
+++ b/test/voice-broadcast/models/VoiceBroadcastRecording-test.ts
@@ -31,7 +31,7 @@ import { Relations } from "matrix-js-sdk/src/models/relations";
 import { SyncState } from "matrix-js-sdk/src/sync";

 import { uploadFile } from "../../../src/ContentMessages";
-import { IEncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
+import { EncryptedFile } from "../../../src/customisations/models/IMediaEventContent";
 import { createVoiceMessageContent } from "../../../src/utils/createVoiceMessageContent";
 import {
    createVoiceBroadcastRecorder,
@@ -82,7 +82,7 @@ jest.mock("../../../src/utils/createVoiceMessageContent", () => ({
 describe("VoiceBroadcastRecording", () => {
    const roomId = "!room:example.com";
    const uploadedUrl = "mxc://example.com/vb";
-    const uploadedFile = { file: true } as unknown as IEncryptedFile;
+    const uploadedFile = { file: true } as unknown as EncryptedFile;
    const maxLength = getMaxBroadcastLength();
    let room: Room;
    let client: MatrixClient;
@@ -223,7 +223,7 @@ describe("VoiceBroadcastRecording", () => {
                mimetype: string,
                duration: number,
                size: number,
-                file?: IEncryptedFile,
+                file?: EncryptedFile,
                waveform?: number[],
            ) => {
                return {