matrix/element-web
1
0
Fork 0
mirror of https://github.com/element-hq/element-web.git synced 2025-08-08 03:42:14 +03:00
Code Activity

Fix more flaky playwright tests (#29007)

Browse Source 
* Group systemic playwright flakes

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix more flaky tests

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix another flake

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* delint

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix more flakes

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix skip tests being wrong

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Michael Telatynski
2025-01-17 11:08:49 +00:00
committed by GitHub
parent 7d30413178
commit e42ee727b4
14 changed files with 104 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
playwright/e2e/audio-player/audio-player.spec.ts
View File

@@ -13,6 +13,14 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
import { Layout } from "../../../src/settings/enums/Layout"; import { Layout } from "../../../src/settings/enums/Layout";
import { ElementAppPage } from "../../pages/ElementAppPage"; import { ElementAppPage } from "../../pages/ElementAppPage";
// Find and click "Reply" button
const clickButtonReply = async (tile: Locator) => {
await expect(async () => {
await tile.hover();
await tile.getByRole("button", { name: "Reply", exact: true }).click();
}).toPass();
};
test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.use({ test.use({
displayName: "Hanako", displayName: "Hanako",
@@ -222,8 +230,7 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
// Find and click "Reply" button on MessageActionBar // Find and click "Reply" button on MessageActionBar
const tile = page.locator(".mx_EventTile_last"); const tile = page.locator(".mx_EventTile_last");
await tile.hover(); await clickButtonReply(tile);
await tile.getByRole("button", { name: "Reply", exact: true }).click();
// Reply to the player with another audio file // Reply to the player with another audio file
await uploadFile(page, "playwright/sample-files/1sec.ogg"); await uploadFile(page, "playwright/sample-files/1sec.ogg");
@@ -251,18 +258,12 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
const tile = page.locator(".mx_EventTile_last"); const tile = page.locator(".mx_EventTile_last");
// Find and click "Reply" button
const clickButtonReply = async () => {
await tile.hover();
await tile.getByRole("button", { name: "Reply", exact: true }).click();
};
await uploadFile(page, "playwright/sample-files/upload-first.ogg"); await uploadFile(page, "playwright/sample-files/upload-first.ogg");
// Assert that the audio player is rendered // Assert that the audio player is rendered
await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible(); await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible();
await clickButtonReply(); await clickButtonReply(tile);
// Reply to the player with another audio file // Reply to the player with another audio file
await uploadFile(page, "playwright/sample-files/upload-second.ogg"); await uploadFile(page, "playwright/sample-files/upload-second.ogg");
@@ -270,7 +271,7 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
// Assert that the audio player is rendered // Assert that the audio player is rendered
await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible(); await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible();
await clickButtonReply(); await clickButtonReply(tile);
// Reply to the player with yet another audio file to create a reply chain // Reply to the player with yet another audio file to create a reply chain
await uploadFile(page, "playwright/sample-files/upload-third.ogg"); await uploadFile(page, "playwright/sample-files/upload-third.ogg");

1
playwright/e2e/knock/create-knock-room.spec.ts
View File

@@ -81,6 +81,7 @@ test.describe("Create Knock Room", () => {
const spotlightDialog = await app.openSpotlight(); const spotlightDialog = await app.openSpotlight();
await spotlightDialog.filter(Filter.PublicRooms); await spotlightDialog.filter(Filter.PublicRooms);
await spotlightDialog.search("Cyber");
await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity"); await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity");
}); });
}); });

1
playwright/e2e/knock/knock-into-room.spec.ts
View File

@@ -284,6 +284,7 @@ test.describe("Knock Into Room", () => {
const spotlightDialog = await app.openSpotlight(); const spotlightDialog = await app.openSpotlight();
await spotlightDialog.filter(Filter.PublicRooms); await spotlightDialog.filter(Filter.PublicRooms);
await spotlightDialog.search("Cyber");
await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity"); await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity");
await spotlightDialog.results.nth(0).click(); await spotlightDialog.results.nth(0).click();

66
playwright/e2e/messages/messages.spec.ts
View File

@@ -58,6 +58,16 @@ async function editMessage(page: Page, message: Locator, newMsg: string): Promis
await editComposer.press("Enter"); await editComposer.press("Enter");
} }
const screenshotOptions = (page?: Page) => ({
mask: page ? [page.locator(".mx_MessageTimestamp")] : undefined,
// Hide the jump to bottom button in the timeline to avoid flakiness
css: `
.mx_JumpToBottomButton {
display: none !important;
}
`,
});
test.describe("Message rendering", () => { test.describe("Message rendering", () => {
[ [
{ direction: "ltr", displayName: "Quentin" }, { direction: "ltr", displayName: "Quentin" },
@@ -79,9 +89,10 @@ test.describe("Message rendering", () => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "Hello, world!"); const msgTile = await sendMessage(page, "Hello, world!");
await expect(msgTile).toMatchScreenshot(`basic-message-ltr-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `basic-message-ltr-${direction}displayname.png`,
}); screenshotOptions(page),
);
}, },
); );
@@ -89,14 +100,17 @@ test.describe("Message rendering", () => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "/me lays an egg"); const msgTile = await sendMessage(page, "/me lays an egg");
await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`); await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`, screenshotOptions());
}); });
test("should render an LTR rich text emote", async ({ page, user, app, room }) => { test("should render an LTR rich text emote", async ({ page, user, app, room }) => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "/me lays a *free range* egg"); const msgTile = await sendMessage(page, "/me lays a *free range* egg");
await expect(msgTile).toMatchScreenshot(`emote-rich-ltr-${direction}displayname.png`); await expect(msgTile).toMatchScreenshot(
`emote-rich-ltr-${direction}displayname.png`,
screenshotOptions(),
);
}); });
test("should render an edited LTR message", async ({ page, user, app, room }) => { test("should render an edited LTR message", async ({ page, user, app, room }) => {
@@ -106,9 +120,10 @@ test.describe("Message rendering", () => {
await editMessage(page, msgTile, "Hello, universe!"); await editMessage(page, msgTile, "Hello, universe!");
await expect(msgTile).toMatchScreenshot(`edited-message-ltr-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `edited-message-ltr-${direction}displayname.png`,
}); screenshotOptions(page),
);
}); });
test("should render a reply of a LTR message", async ({ page, user, app, room }) => { test("should render a reply of a LTR message", async ({ page, user, app, room }) => {
@@ -122,32 +137,37 @@ test.describe("Message rendering", () => {
]); ]);
await replyMessage(page, msgTile, "response to multiline message"); await replyMessage(page, msgTile, "response to multiline message");
await expect(msgTile).toMatchScreenshot(`reply-message-ltr-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `reply-message-ltr-${direction}displayname.png`,
}); screenshotOptions(page),
);
}); });
test("should render a basic RTL text message", async ({ page, user, app, room }) => { test("should render a basic RTL text message", async ({ page, user, app, room }) => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "مرحبا بالعالم!"); const msgTile = await sendMessage(page, "مرحبا بالعالم!");
await expect(msgTile).toMatchScreenshot(`basic-message-rtl-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `basic-message-rtl-${direction}displayname.png`,
}); screenshotOptions(page),
);
}); });
test("should render an RTL emote", async ({ page, user, app, room }) => { test("should render an RTL emote", async ({ page, user, app, room }) => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "/me يضع بيضة"); const msgTile = await sendMessage(page, "/me يضع بيضة");
await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`); await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`, screenshotOptions());
}); });
test("should render a richtext RTL emote", async ({ page, user, app, room }) => { test("should render a richtext RTL emote", async ({ page, user, app, room }) => {
await page.goto(`#/room/${room.roomId}`); await page.goto(`#/room/${room.roomId}`);
const msgTile = await sendMessage(page, "/me أضع بيضة *حرة النطاق*"); const msgTile = await sendMessage(page, "/me أضع بيضة *حرة النطاق*");
await expect(msgTile).toMatchScreenshot(`emote-rich-rtl-${direction}displayname.png`); await expect(msgTile).toMatchScreenshot(
`emote-rich-rtl-${direction}displayname.png`,
screenshotOptions(),
);
}); });
test("should render an edited RTL message", async ({ page, user, app, room }) => { test("should render an edited RTL message", async ({ page, user, app, room }) => {
@@ -157,9 +177,10 @@ test.describe("Message rendering", () => {
await editMessage(page, msgTile, "مرحبا بالكون!"); await editMessage(page, msgTile, "مرحبا بالكون!");
await expect(msgTile).toMatchScreenshot(`edited-message-rtl-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `edited-message-rtl-${direction}displayname.png`,
}); screenshotOptions(page),
);
}); });
test("should render a reply of a RTL message", async ({ page, user, app, room }) => { test("should render a reply of a RTL message", async ({ page, user, app, room }) => {
@@ -173,9 +194,10 @@ test.describe("Message rendering", () => {
]); ]);
await replyMessage(page, msgTile, "مرحبا بالعالم!"); await replyMessage(page, msgTile, "مرحبا بالعالم!");
await expect(msgTile).toMatchScreenshot(`reply-message-trl-${direction}displayname.png`, { await expect(msgTile).toMatchScreenshot(
mask: [page.locator(".mx_MessageTimestamp")], `reply-message-trl-${direction}displayname.png`,
}); screenshotOptions(page),
);
}); });
}); });
}); });

8
playwright/e2e/pinned-messages/pinned-messages.spec.ts
View File

@@ -35,10 +35,10 @@ test.describe("Pinned messages", () => {
mask: [tile.locator(".mx_MessageTimestamp")], mask: [tile.locator(".mx_MessageTimestamp")],
// Hide the jump to bottom button in the timeline to avoid flakiness // Hide the jump to bottom button in the timeline to avoid flakiness
css: ` css: `
.mx_JumpToBottomButton { .mx_JumpToBottomButton {
display: none !important; display: none !important;
} }
`, `,
}); });
}, },
); );

3
playwright/e2e/settings/encryption-user-tab/index.ts
View File

@@ -89,8 +89,7 @@ class Helpers {
await expect(dialog.getByText(title, { exact: true })).toBeVisible(); await expect(dialog.getByText(title, { exact: true })).toBeVisible();
await expect(dialog).toMatchScreenshot(screenshot); await expect(dialog).toMatchScreenshot(screenshot);
const handle = await this.page.evaluateHandle(() => navigator.clipboard.readText()); const clipboardContent = await this.app.getClipboard();
const clipboardContent = await handle.jsonValue();
await dialog.getByRole("textbox").fill(clipboardContent); await dialog.getByRole("textbox").fill(clipboardContent);
await dialog.getByRole("button", { name: confirmButtonLabel }).click(); await dialog.getByRole("button", { name: confirmButtonLabel }).click();
await expect(dialog).toMatchScreenshot("default-recovery.png"); await expect(dialog).toMatchScreenshot("default-recovery.png");

4
playwright/e2e/settings/encryption-user-tab/recovery.spec.ts
View File

@@ -53,7 +53,7 @@ test.describe("Recovery section in Encryption tab", () => {
test( test(
"should change the recovery key", "should change the recovery key",
{ tag: "@screenshot" }, { tag: ["@screenshot", "@no-webkit"] },
async ({ page, app, homeserver, credentials, util, context }) => { async ({ page, app, homeserver, credentials, util, context }) => {
await verifySession(app, "new passphrase"); await verifySession(app, "new passphrase");
const dialog = await util.openEncryptionTab(); const dialog = await util.openEncryptionTab();
@@ -81,7 +81,7 @@ test.describe("Recovery section in Encryption tab", () => {
}, },
); );
test("should setup the recovery key", { tag: "@screenshot" }, async ({ page, app, util }) => { test("should setup the recovery key", { tag: ["@screenshot", "@no-webkit"] }, async ({ page, app, util }) => {
await verifySession(app, "new passphrase"); await verifySession(app, "new passphrase");
await util.removeSecretStorageDefaultKeyId(); await util.removeSecretStorageDefaultKeyId();

4
playwright/e2e/spaces/spaces.spec.ts
View File

@@ -84,7 +84,7 @@ test.describe("Spaces", () => {
// Copy matrix.to link // Copy matrix.to link
await page.getByRole("button", { name: "Share invite link" }).click(); await page.getByRole("button", { name: "Share invite link" }).click();
expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`); expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`);
// Go to space home // Go to space home
await page.getByRole("button", { name: "Go to my first room" }).click(); await page.getByRole("button", { name: "Go to my first room" }).click();
@@ -177,7 +177,7 @@ test.describe("Spaces", () => {
const shareDialog = page.locator(".mx_SpacePublicShare"); const shareDialog = page.locator(".mx_SpacePublicShare");
// Copy link first // Copy link first
await shareDialog.getByRole("button", { name: "Share invite link" }).click(); await shareDialog.getByRole("button", { name: "Share invite link" }).click();
expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`); expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`);
// Start Matrix invite flow // Start Matrix invite flow
await shareDialog.getByRole("button", { name: "Invite people" }).click(); await shareDialog.getByRole("button", { name: "Invite people" }).click();

2
playwright/e2e/spaces/threads-activity-centre/index.ts
View File

@@ -38,11 +38,13 @@ export const test = base.extend<{
room1Name: "Room 1", room1Name: "Room 1",
room1: async ({ room1Name: name, app, user, bot }, use) => { room1: async ({ room1Name: name, app, user, bot }, use) => {
const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] });
await bot.awaitRoomMembership(roomId);
await use({ name, roomId }); await use({ name, roomId });
}, },
room2Name: "Room 2", room2Name: "Room 2",
room2: async ({ room2Name: name, app, user, bot }, use) => { room2: async ({ room2Name: name, app, user, bot }, use) => {
const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] });
await bot.awaitRoomMembership(roomId);
await use({ name, roomId }); await use({ name, roomId });
}, },
msg: async ({ page, app, util }, use) => { msg: async ({ page, app, util }, use) => {

1
playwright/e2e/timeline/timeline.spec.ts
View File

@@ -1195,6 +1195,7 @@ test.describe("Timeline", () => {
}); });
await sendImage(app.client, room.roomId, NEW_AVATAR); await sendImage(app.client, room.roomId, NEW_AVATAR);
await app.timeline.scrollToBottom();
await expect(page.locator(".mx_MImageBody").first()).toBeVisible(); await expect(page.locator(".mx_MImageBody").first()).toBeVisible();
// Exclude timestamp and read marker from snapshot // Exclude timestamp and read marker from snapshot

30
playwright/flaky-reporter.ts
View File

@@ -24,18 +24,40 @@ type PaginationLinks = {
first?: string; first?: string;
}; };
// We see quite a few test flakes which are caused by the app exploding
// so we have some magic strings we check the logs for to better track the flake with its cause
const SPECIAL_CASES = {
"ChunkLoadError": "ChunkLoadError",
"Unreachable code should not be executed": "Rust crypto panic",
"Out of bounds memory access": "Rust crypto memory error",
};
class FlakyReporter implements Reporter { class FlakyReporter implements Reporter {
private flakes = new Map<string, TestCase[]>(); private flakes = new Map<string, TestCase[]>();
public onTestEnd(test: TestCase): void { public onTestEnd(test: TestCase): void {
// Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track // Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track
if (["Dendrite", "Pinecone"].includes(test.parent.project()?.name)) return; if (["Dendrite", "Pinecone"].includes(test.parent.project()?.name)) return;
const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`; let failures = [`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`];
if (test.outcome() === "flaky") { if (test.outcome() === "flaky") {
if (!this.flakes.has(title)) { const timedOutRuns = test.results.filter((result) => result.status === "timedOut");
this.flakes.set(title, []); const pageLogs = timedOutRuns.flatMap((result) =>
result.attachments.filter((attachment) => attachment.name.startsWith("page-")),
);
// If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such.
const specialCases = Object.keys(SPECIAL_CASES).filter((log) =>
pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body.includes(log)),
);
if (specialCases.length > 0) {
failures = specialCases.map((specialCase) => SPECIAL_CASES[specialCase]);
}
for (const title of failures) {
if (!this.flakes.has(title)) {
this.flakes.set(title, []);
}
this.flakes.get(title).push(test);
} }
this.flakes.get(title).push(test);
} }
} }

4
playwright/pages/ElementAppPage.ts
View File

@@ -158,10 +158,6 @@ export class ElementAppPage {
return button.click(); return button.click();
} }
public async getClipboardText(): Promise<string> {
return this.page.evaluate("navigator.clipboard.readText()");
}
public async openSpotlight(): Promise<Spotlight> { public async openSpotlight(): Promise<Spotlight> {
const spotlight = new Spotlight(this.page); const spotlight = new Spotlight(this.page);
await spotlight.open(); await spotlight.open();

16
playwright/pages/client.ts
View File

@@ -15,7 +15,6 @@ import type {
ICreateRoomOpts, ICreateRoomOpts,
ISendEventResponse, ISendEventResponse,
MatrixClient, MatrixClient,
Room,
MatrixEvent, MatrixEvent,
ReceiptType, ReceiptType,
IRoomDirectoryOptions, IRoomDirectoryOptions,
@@ -178,21 +177,12 @@ export class Client {
*/ */
public async createRoom(options: ICreateRoomOpts): Promise<string> { public async createRoom(options: ICreateRoomOpts): Promise<string> {
const client = await this.prepareClient(); const client = await this.prepareClient();
return await client.evaluate(async (cli, options) => { const roomId = await client.evaluate(async (cli, options) => {
const { room_id: roomId } = await cli.createRoom(options); const { room_id: roomId } = await cli.createRoom(options);
if (!cli.getRoom(roomId)) {
await new Promise<void>((resolve) => {
const onRoom = (room: Room) => {
if (room.roomId === roomId) {
cli.off(window.matrixcs.ClientEvent.Room, onRoom);
resolve();
}
};
cli.on(window.matrixcs.ClientEvent.Room, onRoom);
});
}
return roomId; return roomId;
}, options); }, options);
await this.awaitRoomMembership(roomId);
return roomId;
} }
/** /**

8
playwright/services.ts
View File

@@ -155,9 +155,13 @@ export const test = base.extend<TestFixtures, Services & Options>({
{ scope: "worker" }, { scope: "worker" },
], ],
context: async ({ homeserverType, synapseConfig, logger, context, request, homeserver }, use, testInfo) => { context: async (
{ homeserverType, synapseConfig, logger, context, request, _homeserver, homeserver },
use,
testInfo,
) => {
testInfo.skip( testInfo.skip(
!(homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0, !(_homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0,
`Test specifies Synapse config options so is unsupported with ${homeserverType}`, `Test specifies Synapse config options so is unsupported with ${homeserverType}`,
); );
homeserver.setRequest(request); homeserver.setRequest(request);