Fix flaky playwright tests (#28984)

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

playwright/e2e/crypto/backups-mas.spec.ts

@@ -19,19 +19,19 @@ test.use(masHomeserver);
 test.describe("Encryption state after registration", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => {
+    test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!");
 
         await app.settings.openUserSettings("Security & Privacy");
         await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
     });
 
-    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => {
+    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => {
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!");
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!");
 
         await page.getByRole("button", { name: "Add room" }).click();
         await page.getByRole("menuitem", { name: "New room" }).click();
@@ -45,8 +45,13 @@ test.describe("Encryption state after registration", () => {
 test.describe("Key backup reset from elsewhere", () => {
     test.skip(isDendrite, "does not yet support MAS");
 
-    test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => {
+    test("Key backup is disabled when reset from elsewhere", async ({
-        const testUsername = "alice";
+        page,
+        mailhogClient,
+        request,
+        homeserver,
+    }, testInfo) => {
+        const testUsername = `alice_${testInfo.testId}`;
         const testPassword = "Pa$sW0rD!";
 
         // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake

playwright/e2e/login/login-sso.spec.ts

@@ -17,13 +17,13 @@ test.use(legacyOAuthHomeserver);
 test.describe("SSO login", () => {
     test.skip(isDendrite, "does not yet support SSO");
 
-    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => {
+    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => {
         // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
         // your firewall settings: Synapse is unable to reach the OIDC server.
         //
         // If you are using ufw, try something like:
         //    sudo ufw allow in on docker0
         //
-        await doTokenRegistration(page, homeserver);
+        await doTokenRegistration(page, homeserver, testInfo);
     });
 });

playwright/e2e/login/soft_logout_oauth.spec.ts

@@ -26,8 +26,8 @@ test.use({
 test.use(legacyOAuthHomeserver);
 test.describe("Soft logout with SSO user", () => {
     test.use({
-        user: async ({ page, homeserver }, use) => {
+        user: async ({ page, homeserver }, use, testInfo) => {
-            const user = await doTokenRegistration(page, homeserver);
+            const user = await doTokenRegistration(page, homeserver, testInfo);
 
             // Eventually, we should end up at the home screen.
             await expect(page).toHaveURL(/\/#\/home$/);

playwright/e2e/login/utils.ts

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Page, expect } from "@playwright/test";
+import { Page, expect, TestInfo } from "@playwright/test";
 
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 
@@ -15,6 +15,7 @@ import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 export async function doTokenRegistration(
     page: Page,
     homeserver: HomeserverInstance,
+    testInfo: TestInfo,
 ): Promise<Credentials & { displayName: string }> {
     await page.goto("/#/login");
 
@@ -35,7 +36,7 @@ export async function doTokenRegistration(
 
     // Synapse prompts us to pick a user ID
     await expect(page.getByRole("heading", { name: "Create your account" })).toBeVisible();
-    await page.getByRole("textbox", { name: "Username (required)" }).fill("alice");
+    await page.getByRole("textbox", { name: "Username (required)" }).fill(`alice_${testInfo.testId}`);
 
     // wait for username validation to start, and complete
     await expect(page.locator("#field-username-output")).toHaveText("");

playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts

@@ -6,20 +6,31 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures } from "@playwright/test";
+import { Fixtures, PlaywrightTestArgs } from "@playwright/test";
 import { TestContainers } from "testcontainers";
 
 import { Services } from "../../../services.ts";
 import { OAuthServer } from "../../oauth_server";
 
-export const legacyOAuthHomeserver: Fixtures<{}, Services> = {
+export const legacyOAuthHomeserver: Fixtures<PlaywrightTestArgs, Services, PlaywrightTestArgs> = {
-    _homeserver: [
+    oAuthServer: [
-        async ({ _homeserver: container }, use) => {
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
             const server = new OAuthServer();
-            const port = server.start();
+            await use(server);
-
+            server.stop();
+        },
+        { scope: "worker" },
+    ],
+    context: async ({ context, oAuthServer }, use, testInfo) => {
+        oAuthServer.onTestStarted(testInfo);
+        await use(context);
+    },
+    _homeserver: [
+        async ({ oAuthServer, _homeserver: homeserver }, use) => {
+            const port = oAuthServer.start();
             await TestContainers.exposeHostPorts(port);
-            container.withConfig({
+            homeserver.withConfig({
                 oidc_providers: [
                     {
                         idp_id: "test",
@@ -43,8 +54,8 @@ export const legacyOAuthHomeserver: Fixtures<{}, Services> = {
                     },
                 ],
             });
-            await use(container);
+
-            server.stop();
+            await use(homeserver);
         },
         { scope: "worker" },
     ],

playwright/plugins/oauth_server/index.ts

@@ -9,12 +9,21 @@ Please see LICENSE files in the repository root for full details.
 import http from "http";
 import express from "express";
 import { AddressInfo } from "net";
+import { TestInfo } from "@playwright/test";
+
+import { randB64Bytes } from "../utils/rand.ts";
 
 export class OAuthServer {
     private server?: http.Server;
+    private sub?: string;
+
+    public onTestStarted(testInfo: TestInfo): void {
+        this.sub = testInfo.testId;
+    }
 
     public start(): number {
         if (this.server) this.stop();
+        const token = randB64Bytes(16);
 
         const app = express();
 
@@ -28,7 +37,7 @@ export class OAuthServer {
             const code = req.body.code;
             if (code === "valid_auth_code") {
                 res.send({
-                    access_token: "oauth_access_token",
+                    access_token: token,
                     token_type: "Bearer",
                     expires_in: "3600",
                 });
@@ -43,7 +52,7 @@ export class OAuthServer {
 
             // return an OAuth2 user info object
             res.send({
-                sub: "alice",
+                sub: this.sub,
                 name: "Alice",
             });
         });

playwright/services.ts

@@ -15,6 +15,7 @@ import { Logger } from "./logger.ts";
 import { StartedMatrixAuthenticationServiceContainer } from "./testcontainers/mas.ts";
 import { HomeserverContainer, StartedHomeserverContainer } from "./testcontainers/HomeserverContainer.ts";
 import { MailhogContainer, StartedMailhogContainer } from "./testcontainers/mailhog.ts";
+import { OAuthServer } from "./plugins/oauth_server";
 
 interface TestFixtures {
     mailhogClient: mailhog.API;
@@ -30,7 +31,10 @@ export interface Services {
     synapseConfigOptions: SynapseConfigOptions;
     _homeserver: HomeserverContainer<any>;
     homeserver: StartedHomeserverContainer;
+    // Set in masHomeserver only
     mas?: StartedMatrixAuthenticationServiceContainer;
+    // Set in legacyOAuthHomeserver only
+    oAuthServer?: OAuthServer;
 }
 
 export const test = base.extend<TestFixtures, Services>({