refactor(test): improve test scenario reliability and maintainability (#3077)

* refactor(test): improve test scenario reliability and maintainability

* tests: add resp3 check test (#1)

* test: refactor connection handoff tests with enhanced spy utility (#2)

* test: add comprehensive push notification disabled scenarios (#3)

* tests: add params config tests (#4)

* tests: add feature enablement tests (#5)

---------

Co-authored-by: Nikolay Karadzhov <nikolay.karadzhov@redis.com>

packages/client/lib/client/enterprise-maintenance-manager.ts

@@ -51,9 +51,10 @@ interface Client {
   _pause: () => void;
   _unpause: () => void;
   _maintenanceUpdate: (update: MaintenanceUpdate) => void;
-  duplicate: (options: RedisClientOptions) => Client;
+  duplicate: () => Client;
   connect: () => Promise<Client>;
   destroy: () => void;
+  on: (event: string, callback: (value: unknown) => void) => void;
 }
 
 export default class EnterpriseMaintenanceManager {
@@ -211,21 +212,25 @@ export default class EnterpriseMaintenanceManager {
     dbgMaintenance("Creating new tmp client");
     let start = performance.now();
 
-    const tmpOptions = this.#options;
     // If the URL is provided, it takes precedense
-    if(tmpOptions.url) {
+    // the options object could just be mutated
-      const u = new URL(tmpOptions.url);
+    if(this.#options.url) {
+      const u = new URL(this.#options.url);
       u.hostname = host;
       u.port = String(port);
-      tmpOptions.url = u.toString();
+      this.#options.url = u.toString();
     } else {
-      tmpOptions.socket = {
+      this.#options.socket = {
-        ...tmpOptions.socket,
+        ...this.#options.socket,
         host,
         port
       }
     }
-    const tmpClient = this.#client.duplicate(tmpOptions);
+    const tmpClient = this.#client.duplicate();
+    tmpClient.on('error', (error: unknown) => {
+      //We dont know how to handle tmp client errors
+      dbgMaintenance(`[ERR]`, error)
+    });
     dbgMaintenance(`Tmp client created in ${( performance.now() - start ).toFixed(2)}ms`);
     dbgMaintenance(
       `Set timeout for tmp client to ${this.#options.maintRelaxedSocketTimeout}`,

packages/client/lib/tests/test-scenario/configuration.e2e.ts

@@ -0,0 +1,201 @@
+import assert from "node:assert";
+import diagnostics_channel from "node:diagnostics_channel";
+import { DiagnosticsEvent } from "../../client/enterprise-maintenance-manager";
+
+import {
+  RedisConnectionConfig,
+  createTestClient,
+  getDatabaseConfig,
+  getDatabaseConfigFromEnv,
+  getEnvConfig,
+} from "./test-scenario.util";
+import { createClient } from "../../..";
+import { FaultInjectorClient } from "./fault-injector-client";
+import { MovingEndpointType } from "../../../dist/lib/client/enterprise-maintenance-manager";
+import { RedisTcpSocketOptions } from "../../client/socket";
+
+describe("Client Configuration and Handshake", () => {
+  let clientConfig: RedisConnectionConfig;
+  let client: ReturnType<typeof createClient<any, any, any, any>>;
+  let faultInjectorClient: FaultInjectorClient;
+  let log: DiagnosticsEvent[] = [];
+
+  before(() => {
+    const envConfig = getEnvConfig();
+    const redisConfig = getDatabaseConfigFromEnv(
+      envConfig.redisEndpointsConfigPath,
+    );
+
+    faultInjectorClient = new FaultInjectorClient(envConfig.faultInjectorUrl);
+    clientConfig = getDatabaseConfig(redisConfig);
+
+    diagnostics_channel.subscribe("redis.maintenance", (event) => {
+      log.push(event as DiagnosticsEvent);
+    });
+  });
+
+  beforeEach(() => {
+    log.length = 0;
+  });
+
+  afterEach(async () => {
+    if (client && client.isOpen) {
+      await client.flushAll();
+      client.destroy();
+    }
+  });
+
+  describe("Parameter Configuration", () => {
+    const endpoints: MovingEndpointType[] = [
+      "auto",
+      // "internal-ip",
+      // "internal-fqdn",
+      "external-ip",
+      "external-fqdn",
+      "none",
+    ];
+
+    for (const endpointType of endpoints) {
+      it(`clientHandshakeWithEndpointType '${endpointType}'`, async () => {
+        try {
+          client = await createTestClient(clientConfig, {
+            maintMovingEndpointType: endpointType,
+          });
+          client.on("error", () => {});
+
+          //need to copy those because they will be mutated later
+          const oldOptions = JSON.parse(JSON.stringify(client.options));
+          assert.ok(oldOptions);
+
+          const { action_id } = await faultInjectorClient.migrateAndBindAction({
+            bdbId: clientConfig.bdbId,
+            clusterIndex: 0,
+          });
+
+          await faultInjectorClient.waitForAction(action_id);
+
+          const movingEvent = log.find((event) => event.type === "MOVING");
+          assert(!!movingEvent, "Didnt receive moving PN");
+
+          let endpoint: string | undefined;
+          try {
+            //@ts-ignore
+            endpoint = movingEvent.data.push[3];
+          } catch (err) {
+            assert(
+              false,
+              `couldnt get endpoint from event ${JSON.stringify(movingEvent)}`,
+            );
+          }
+
+          assert(endpoint !== undefined, "no endpoint");
+
+          const newOptions = client.options;
+          assert.ok(newOptions);
+
+          if (oldOptions?.url) {
+            if (endpointType === "none") {
+              assert.equal(
+                newOptions!.url,
+                oldOptions.url,
+                "For movingEndpointTpe 'none', we expect old and new url to be the same",
+              );
+            } else {
+              assert.equal(
+                newOptions.url,
+                endpoint,
+                "Expected what came through the wire to be set in the new client",
+              );
+              assert.notEqual(
+                newOptions!.url,
+                oldOptions.url,
+                `For movingEndpointTpe ${endpointType}, we expect old and new url to be different`,
+              );
+            }
+          } else {
+            const oldSocket = oldOptions.socket as RedisTcpSocketOptions;
+            const newSocket = newOptions.socket as RedisTcpSocketOptions;
+            assert.ok(oldSocket);
+            assert.ok(newSocket);
+
+            if (endpointType === "none") {
+              assert.equal(
+                newSocket.host,
+                oldSocket.host,
+                "For movingEndpointTpe 'none', we expect old and new host to be the same",
+              );
+            } else {
+              assert.equal(
+                newSocket.host + ":" + newSocket.port,
+                endpoint,
+                "Expected what came through the wire to be set in the new client",
+              );
+              assert.notEqual(
+                newSocket.host,
+                oldSocket.host,
+                `For movingEndpointTpe ${endpointType}, we expect old and new host to be different`,
+              );
+            }
+          }
+        } catch (error: any) {
+          if (
+            endpointType === "internal-fqdn" ||
+            endpointType === "internal-ip"
+          ) {
+            // errors are expected here, because we cannot connect to internal endpoints unless we are deployed in the same place as the server
+          } else {
+            assert(false, error);
+          }
+        }
+      });
+    }
+  });
+
+  describe("Feature Enablement", () => {
+    it("connectionHandshakeIncludesEnablingNotifications", async () => {
+      client = await createTestClient(clientConfig, {
+        maintPushNotifications: "enabled",
+      });
+
+      const { action_id } = await faultInjectorClient.migrateAndBindAction({
+        bdbId: clientConfig.bdbId,
+        clusterIndex: 0,
+      });
+
+      await faultInjectorClient.waitForAction(action_id);
+
+      let movingEvent = false;
+      let migratingEvent = false;
+      let migratedEvent = false;
+      for (const event of log) {
+        if (event.type === "MOVING") movingEvent = true;
+        if (event.type === "MIGRATING") migratingEvent = true;
+        if (event.type === "MIGRATED") migratedEvent = true;
+      }
+      assert.ok(movingEvent, "didnt receive MOVING PN");
+      assert.ok(migratingEvent, "didnt receive MIGRATING PN");
+      assert.ok(migratedEvent, "didnt receive MIGRATED PN");
+    });
+
+    it("disabledDontReceiveNotifications", async () => {
+      try {
+        client = await createTestClient(clientConfig, {
+          maintPushNotifications: "disabled",
+          socket: {
+            reconnectStrategy: false
+          }
+        });
+        client.on('error', console.log.bind(console))
+
+        const { action_id } = await faultInjectorClient.migrateAndBindAction({
+          bdbId: clientConfig.bdbId,
+          clusterIndex: 0,
+        });
+
+        await faultInjectorClient.waitForAction(action_id);
+
+        assert.equal(log.length, 0, "received a PN while feature is disabled");
+      } catch (error: any) { }
+    });
+  });
+});

packages/client/lib/tests/test-scenario/connection-handoff.e2e.ts

@@ -1,78 +1,162 @@
-import diagnostics_channel from "node:diagnostics_channel";
 import { FaultInjectorClient } from "./fault-injector-client";
 import {
+  createTestClient,
   getDatabaseConfig,
   getDatabaseConfigFromEnv,
   getEnvConfig,
   RedisConnectionConfig,
 } from "./test-scenario.util";
-import { createClient } from "../../..";
+import { createClient, RedisClientOptions } from "../../..";
-import { DiagnosticsEvent } from "../../client/enterprise-maintenance-manager";
 import { before } from "mocha";
-import { spy } from "sinon";
+import Sinon, { SinonSpy, spy, stub } from "sinon";
 import assert from "node:assert";
-import { TestCommandRunner } from "./test-command-runner";
+
-import net from "node:net";
+/**
+ * Creates a spy on a duplicated client method
+ * @param client - The Redis client instance
+ * @param funcName - The name of the method to spy on
+ * @returns Object containing the promise that resolves with the spy and restore function
+ */
+const spyOnTemporaryClientInstanceMethod = (
+  client: ReturnType<typeof createClient<any, any, any, any>>,
+  methodName: string
+) => {
+  const { promise, resolve } = (
+    Promise as typeof Promise & {
+      withResolvers: () => {
+        promise: Promise<{ spy: SinonSpy<any[], any>; restore: () => void }>;
+        resolve: (value: any) => void;
+      };
+    }
+  ).withResolvers();
+
+  const originalDuplicate = client.duplicate.bind(client);
+
+  const duplicateStub: Sinon.SinonStub<any[], any> = stub(
+    // Temporary clients (in the context of hitless upgrade)
+    // are created by calling the duplicate method on the client.
+    Object.getPrototypeOf(client),
+    "duplicate"
+  ).callsFake((opts) => {
+    const tmpClient = originalDuplicate(opts);
+    resolve({
+      spy: spy(tmpClient, methodName),
+      restore: duplicateStub.restore,
+    });
+
+    return tmpClient;
+  });
+
+  return {
+    getSpy: () => promise,
+  };
+};
 
 describe("Connection Handoff", () => {
-  const diagnosticsLog: DiagnosticsEvent[] = [];
-
-  const onMessageHandler = (message: unknown) => {
-    diagnosticsLog.push(message as DiagnosticsEvent);
-  };
-
   let clientConfig: RedisConnectionConfig;
-  let client: ReturnType<typeof createClient<any, any, any, 3>>;
+  let client: ReturnType<typeof createClient<any, any, any, any>>;
   let faultInjectorClient: FaultInjectorClient;
-  let connectSpy = spy(net, "createConnection");
 
   before(() => {
     const envConfig = getEnvConfig();
     const redisConfig = getDatabaseConfigFromEnv(
-      envConfig.redisEndpointsConfigPath,
+      envConfig.redisEndpointsConfigPath
     );
 
     faultInjectorClient = new FaultInjectorClient(envConfig.faultInjectorUrl);
     clientConfig = getDatabaseConfig(redisConfig);
   });
 
-  beforeEach(async () => {
+  afterEach(async () => {
-    diagnosticsLog.length = 0;
+    if (client && client.isOpen) {
-    diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+      await client.flushAll();
+      client.destroy();
+    }
+  });
 
-    connectSpy.resetHistory();
+  describe("New Connection Establishment & Traffic Resumption", () => {
-
+    const cases: Array<{
-    client = createClient({
+      name: string;
-      socket: {
+      clientOptions: Partial<RedisClientOptions>;
-        host: clientConfig.host,
+    }> = [
-        port: clientConfig.port,
+      {
-        ...(clientConfig.tls === true ? { tls: true } : {}),
+        name: "default options",
+        clientOptions: {},
       },
-      password: clientConfig.password,
+      {
-      username: clientConfig.username,
+        name: "external-ip",
-      RESP: 3,
+        clientOptions: {
-      maintPushNotifications: "auto",
+          maintMovingEndpointType: "external-ip",
-      maintMovingEndpointType: "external-ip",
+        },
-      maintRelaxedCommandTimeout: 10000,
+      },
-      maintRelaxedSocketTimeout: 10000,
+      {
-    });
+        name: "external-fqdn",
+        clientOptions: {
+          maintMovingEndpointType: "external-fqdn",
+        },
+      },
+      {
+        name: "auto",
+        clientOptions: {
+          maintMovingEndpointType: "auto",
+        },
+      },
+      {
+        name: "none",
+        clientOptions: {
+          maintMovingEndpointType: "none",
+        },
+      },
+    ];
 
-    client.on("error", (err: Error) => {
+    for (const { name, clientOptions } of cases) {
-      throw new Error(`Client error: ${err.message}`);
+      it(`should establish new connection and resume traffic afterwards - ${name}`, async () => {
-    });
+        client = await createTestClient(clientConfig, clientOptions);
 
-    await client.connect();
+        const spyObject = spyOnTemporaryClientInstanceMethod(client, "connect");
-    await client.flushAll();
+
+        // PART 1 Establish initial connection
+        const { action_id: lowTimeoutBindAndMigrateActionId } =
+          await faultInjectorClient.migrateAndBindAction({
+            bdbId: clientConfig.bdbId,
+            clusterIndex: 0,
+          });
+
+        await faultInjectorClient.waitForAction(
+          lowTimeoutBindAndMigrateActionId
+        );
+
+        const spyResult = await spyObject.getSpy();
+
+        assert.strictEqual(spyResult.spy.callCount, 1);
+
+        // PART 2 Verify traffic resumption
+        const currentTime = Date.now().toString();
+        await client.set("key", currentTime);
+        const result = await client.get("key");
+
+        assert.strictEqual(result, currentTime);
+
+        spyResult.restore();
+      });
+    }
   });
 
-  afterEach(() => {
+  describe("TLS Connection Handoff", () => {
-    diagnostics_channel.unsubscribe("redis.maintenance", onMessageHandler);
+    it.skip("TODO receiveMessagesWithTLSEnabledTest", async () => {
-    client.destroy();
+      //
+    });
+    it.skip("TODO connectionHandoffWithStaticInternalNameTest", async () => {
+      //
+    });
+    it.skip("TODO connectionHandoffWithStaticExternalNameTest", async () => {
+      //
+    });
   });
 
-  describe("New Connection Establishment", () => {
+  describe("Connection Cleanup", () => {
-    it("should establish new connection", async () => {
+    it("should shut down old connection", async () => {
-      assert.equal(connectSpy.callCount, 1);
+      const spyObject = spyOnTemporaryClientInstanceMethod(client, "destroy");
 
       const { action_id: lowTimeoutBindAndMigrateActionId } =
         await faultInjectorClient.migrateAndBindAction({
@@ -80,47 +164,13 @@ describe("Connection Handoff", () => {
           clusterIndex: 0,
         });
 
-      const lowTimeoutWaitPromise = faultInjectorClient.waitForAction(
+      await faultInjectorClient.waitForAction(lowTimeoutBindAndMigrateActionId);
-        lowTimeoutBindAndMigrateActionId,
-      );
 
-      await lowTimeoutWaitPromise;
+      const spyResult = await spyObject.getSpy();
-      assert.equal(connectSpy.callCount, 2);
-    });
-  });
 
-  describe("TLS Connection Handoff", () => {
+      assert.equal(spyResult.spy.callCount, 1);
-    it("TODO receiveMessagesWithTLSEnabledTest", async () => {
-      //
-    });
-    it("TODO connectionHandoffWithStaticInternalNameTest", async () => {
-      //
-    });
-    it("TODO connectionHandoffWithStaticExternalNameTest", async () => {
-      //
-    });
-  });
 
-  describe("Traffic Resumption", () => {
+      spyResult.restore();
-    it("Traffic resumed after handoff", async () => {
-      const { action_id } = await faultInjectorClient.migrateAndBindAction({
-        bdbId: clientConfig.bdbId,
-        clusterIndex: 0,
-      });
-
-      const workloadPromise = faultInjectorClient.waitForAction(action_id);
-
-      const commandPromises =
-        await TestCommandRunner.fireCommandsUntilStopSignal(
-          client,
-          workloadPromise,
-        );
-
-      const rejected = (
-        await Promise.all(commandPromises.commandPromises)
-      ).filter((result) => result.status === "rejected");
-
-      assert.ok(rejected.length === 0);
     });
   });
 });

packages/client/lib/tests/test-scenario/fault-injector-client.ts

@@ -9,7 +9,8 @@ export type ActionType =
   | "execute_rlutil_command"
   | "execute_rladmin_command"
   | "migrate"
-  | "bind";
+  | "bind"
+  | "update_cluster_config";
 
 export interface ActionRequest {
   type: ActionType;
@@ -47,7 +48,9 @@ export class FaultInjectorClient {
    * @param action The action request to trigger
    * @throws {Error} When the HTTP request fails or response cannot be parsed as JSON
    */
-  public triggerAction<T = unknown>(action: ActionRequest): Promise<T> {
+  public triggerAction<T extends { action_id: string }>(
+    action: ActionRequest
+  ): Promise<T> {
     return this.#request<T>("POST", "/action", action);
   }
 
@@ -60,20 +63,6 @@ export class FaultInjectorClient {
     return this.#request<T>("GET", `/action/${actionId}`);
   }
 
-  /**
-   * Executes an rladmin command.
-   * @param command The rladmin command to execute
-   * @param bdbId Optional database ID to target
-   * @throws {Error} When the HTTP request fails or response cannot be parsed as JSON
-   */
-  public executeRladminCommand<T = unknown>(
-    command: string,
-    bdbId?: string
-  ): Promise<T> {
-    const cmd = bdbId ? `rladmin -b ${bdbId} ${command}` : `rladmin ${command}`;
-    return this.#request<T>("POST", "/rladmin", cmd);
-  }
-
   /**
    * Waits for an action to complete.
    * @param actionId The ID of the action to wait for

packages/client/lib/tests/test-scenario/negative-tests.e2e.ts

@@ -0,0 +1,15 @@
+import assert from "assert";
+import { createClient } from "../../..";
+
+describe("Negative tests", () => {
+  it("should only be enabled with RESP3", () => {
+    assert.throws(
+      () =>
+        createClient({
+          RESP: 2,
+          maintPushNotifications: "enabled",
+        }),
+      "Error: Graceful Maintenance is only supported with RESP3",
+    );
+  });
+});

packages/client/lib/tests/test-scenario/push-notification.e2e.ts

@@ -2,6 +2,7 @@ import assert from "node:assert";
 import diagnostics_channel from "node:diagnostics_channel";
 import { FaultInjectorClient } from "./fault-injector-client";
 import {
+  createTestClient,
   getDatabaseConfig,
   getDatabaseConfigFromEnv,
   getEnvConfig,
@@ -12,14 +13,21 @@ import { DiagnosticsEvent } from "../../client/enterprise-maintenance-manager";
 import { before } from "mocha";
 
 describe("Push Notifications", () => {
-  const diagnosticsLog: DiagnosticsEvent[] = [];
+  const createNotificationMessageHandler = (
-
+    result: Record<DiagnosticsEvent["type"], number>,
-  const onMessageHandler = (message: unknown) => {
+    notifications: Array<DiagnosticsEvent["type"]>
-    diagnosticsLog.push(message as DiagnosticsEvent);
+  ) => {
+    return (message: unknown) => {
+      if (notifications.includes((message as DiagnosticsEvent).type)) {
+        const event = message as DiagnosticsEvent;
+        result[event.type] = (result[event.type] ?? 0) + 1;
+      }
+    };
   };
 
+  let onMessageHandler: ReturnType<typeof createNotificationMessageHandler>;
   let clientConfig: RedisConnectionConfig;
-  let client: ReturnType<typeof createClient<any, any, any, 3>>;
+  let client: ReturnType<typeof createClient<any, any, any, any>>;
   let faultInjectorClient: FaultInjectorClient;
 
   before(() => {
@@ -32,63 +40,310 @@ describe("Push Notifications", () => {
     clientConfig = getDatabaseConfig(redisConfig);
   });
 
-  beforeEach(async () => {
-    diagnosticsLog.length = 0;
-    diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
-
-    client = createClient({
-      socket: {
-        host: clientConfig.host,
-        port: clientConfig.port,
-        ...(clientConfig.tls === true ? { tls: true } : {}),
-      },
-      password: clientConfig.password,
-      username: clientConfig.username,
-      RESP: 3,
-      maintPushNotifications: "auto",
-      maintMovingEndpointType: "external-ip",
-      maintRelaxedCommandTimeout: 10000,
-      maintRelaxedSocketTimeout: 10000,
-    });
-
-    client.on("error", (err: Error) => {
-      throw new Error(`Client error: ${err.message}`);
-    });
-
-    await client.connect();
-  });
-
   afterEach(() => {
-    diagnostics_channel.unsubscribe("redis.maintenance", onMessageHandler);
+    if (onMessageHandler!) {
-    client.destroy();
+      diagnostics_channel.unsubscribe("redis.maintenance", onMessageHandler);
+    }
+
+    if (client && client.isOpen) {
+      client.destroy();
+    }
   });
 
-  it("should receive MOVING, MIGRATING, and MIGRATED push notifications", async () => {
+  describe("Push Notifications Enabled", () => {
-    const { action_id: migrateActionId } =
+    beforeEach(async () => {
-      await faultInjectorClient.triggerAction<{ action_id: string }>({
+      client = await createTestClient(clientConfig);
-        type: "migrate",
-        parameters: {
-          cluster_index: "0",
-        },
-      });
 
-    await faultInjectorClient.waitForAction(migrateActionId);
+      await client.flushAll();
-
-    const { action_id: bindActionId } =
-      await faultInjectorClient.triggerAction<{ action_id: string }>({
-        type: "bind",
-        parameters: {
-          cluster_index: "0",
-          bdb_id: `${clientConfig.bdbId}`,
-        },
-      });
-
-    await faultInjectorClient.waitForAction(bindActionId);
-
-    const pushNotificationLogs = diagnosticsLog.filter((log) => {
-      return ["MOVING", "MIGRATING", "MIGRATED"].includes(log?.type);
     });
 
-    assert.strictEqual(pushNotificationLogs.length, 3);
+    it("should receive MOVING, MIGRATING, and MIGRATED push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "MOVING",
+        "MIGRATING",
+        "MIGRATED",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: bindAndMigrateActionId } =
+        await faultInjectorClient.migrateAndBindAction({
+          bdbId: clientConfig.bdbId,
+          clusterIndex: 0,
+        });
+
+      await faultInjectorClient.waitForAction(bindAndMigrateActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.MOVING,
+        1,
+        "Should have received exactly one MOVING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATING,
+        1,
+        "Should have received exactly one MIGRATING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATED,
+        1,
+        "Should have received exactly one MIGRATED notification"
+      );
+    });
+
+    it("should receive FAILING_OVER and FAILED_OVER push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "FAILING_OVER",
+        "FAILED_OVER",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: failoverActionId } =
+        await faultInjectorClient.triggerAction({
+          type: "failover",
+          parameters: {
+            bdb_id: clientConfig.bdbId.toString(),
+            cluster_index: 0,
+          },
+        });
+
+      await faultInjectorClient.waitForAction(failoverActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.FAILING_OVER,
+        1,
+        "Should have received exactly one FAILING_OVER notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.FAILED_OVER,
+        1,
+        "Should have received exactly one FAILED_OVER notification"
+      );
+    });
+  });
+
+  describe("Push Notifications Disabled - Client", () => {
+    beforeEach(async () => {
+      client = await createTestClient(clientConfig, {
+        maintPushNotifications: "disabled",
+      });
+
+      client.on("error", (_err) => {
+        // Expect the socket to be closed
+        // Ignore errors
+      });
+
+      await client.flushAll();
+    });
+
+    it("should NOT receive MOVING, MIGRATING, and MIGRATED push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "MOVING",
+        "MIGRATING",
+        "MIGRATED",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: bindAndMigrateActionId } =
+        await faultInjectorClient.migrateAndBindAction({
+          bdbId: clientConfig.bdbId,
+          clusterIndex: 0,
+        });
+
+      await faultInjectorClient.waitForAction(bindAndMigrateActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.MOVING,
+        undefined,
+        "Should NOT have received exactly one MOVING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATING,
+        undefined,
+        "Should NOT have received exactly one MIGRATING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATED,
+        undefined,
+        "Should NOT have received exactly one MIGRATED notification"
+      );
+    });
+
+    it("should NOT receive FAILING_OVER and FAILED_OVER push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "FAILING_OVER",
+        "FAILED_OVER",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: failoverActionId } =
+        await faultInjectorClient.triggerAction({
+          type: "failover",
+          parameters: {
+            bdb_id: clientConfig.bdbId.toString(),
+            cluster_index: 0,
+          },
+        });
+
+      await faultInjectorClient.waitForAction(failoverActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.FAILING_OVER,
+        undefined,
+        "Should have received exactly one FAILING_OVER notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.FAILED_OVER,
+        undefined,
+        "Should have received exactly one FAILED_OVER notification"
+      );
+    });
+  });
+
+  describe("Push Notifications Disabled - Server", () => {
+    beforeEach(async () => {
+      client = await createTestClient(clientConfig);
+
+      client.on("error", (_err) => {
+        // Expect the socket to be closed
+        // Ignore errors
+      });
+
+      await client.flushAll();
+    });
+
+    before(async () => {
+      const { action_id: disablePushNotificationsActionId } =
+        await faultInjectorClient.triggerAction({
+          type: "update_cluster_config",
+          parameters: {
+            config: { client_maint_notifications: false },
+          },
+        });
+
+      await faultInjectorClient.waitForAction(disablePushNotificationsActionId);
+    });
+
+    after(async () => {
+      const { action_id: enablePushNotificationsActionId } =
+        await faultInjectorClient.triggerAction({
+          type: "update_cluster_config",
+          parameters: {
+            config: { client_maint_notifications: true },
+          },
+        });
+
+      await faultInjectorClient.waitForAction(enablePushNotificationsActionId);
+    });
+
+    it("should NOT receive MOVING, MIGRATING, and MIGRATED push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "MOVING",
+        "MIGRATING",
+        "MIGRATED",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: bindAndMigrateActionId } =
+        await faultInjectorClient.migrateAndBindAction({
+          bdbId: clientConfig.bdbId,
+          clusterIndex: 0,
+        });
+
+      await faultInjectorClient.waitForAction(bindAndMigrateActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.MOVING,
+        undefined,
+        "Should NOT have received exactly one MOVING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATING,
+        undefined,
+        "Should NOT have received exactly one MIGRATING notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.MIGRATED,
+        undefined,
+        "Should NOT have received exactly one MIGRATED notification"
+      );
+    });
+
+    it("should NOT receive FAILING_OVER and FAILED_OVER push notifications", async () => {
+      const notifications: Array<DiagnosticsEvent["type"]> = [
+        "FAILING_OVER",
+        "FAILED_OVER",
+      ];
+
+      const diagnosticsMap: Record<DiagnosticsEvent["type"], number> = {};
+
+      onMessageHandler = createNotificationMessageHandler(
+        diagnosticsMap,
+        notifications
+      );
+
+      diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+      const { action_id: failoverActionId } =
+        await faultInjectorClient.triggerAction({
+          type: "failover",
+          parameters: {
+            bdb_id: clientConfig.bdbId.toString(),
+            cluster_index: 0,
+          },
+        });
+
+      await faultInjectorClient.waitForAction(failoverActionId);
+
+      assert.strictEqual(
+        diagnosticsMap.FAILING_OVER,
+        undefined,
+        "Should have received exactly one FAILING_OVER notification"
+      );
+      assert.strictEqual(
+        diagnosticsMap.FAILED_OVER,
+        undefined,
+        "Should have received exactly one FAILED_OVER notification"
+      );
+    });
   });
 });

packages/client/lib/tests/test-scenario/test-command-runner.ts

@@ -1,108 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { setTimeout } from "node:timers/promises";
-import { createClient } from "../../..";
-
-/**
- * Options for the `fireCommandsUntilStopSignal` method
- */
-type FireCommandsUntilStopSignalOptions = {
-  /**
-   * Number of commands to fire in each batch
-   */
-  batchSize: number;
-  /**
-   * Timeout between batches in milliseconds
-   */
-  timeoutMs: number;
-  /**
-   * Function that creates the commands to be executed
-   */
-  createCommands: (
-    client: ReturnType<typeof createClient<any, any, any, any>>
-  ) => Array<() => Promise<unknown>>;
-};
-
-/**
- * Utility class for running test commands until a stop signal is received
- */
-export class TestCommandRunner {
-  private static readonly defaultOptions: FireCommandsUntilStopSignalOptions = {
-    batchSize: 60,
-    timeoutMs: 10,
-    createCommands: (
-      client: ReturnType<typeof createClient<any, any, any, any>>
-    ) => [
-      () => client.set(randomUUID(), Date.now()),
-      () => client.get(randomUUID()),
-    ],
-  };
-
-  static #toSettled<T>(p: Promise<T>) {
-    return p
-      .then((value) => ({ status: "fulfilled" as const, value, error: null }))
-      .catch((reason) => ({
-        status: "rejected" as const,
-        value: null,
-        error: reason,
-      }));
-  }
-
-  static async #racePromises<S, T>({
-    timeout,
-    stopper,
-  }: {
-    timeout: Promise<S>;
-    stopper: Promise<T>;
-  }) {
-    return Promise.race([
-      TestCommandRunner.#toSettled<S>(timeout).then((result) => ({
-        ...result,
-        stop: false,
-      })),
-      TestCommandRunner.#toSettled<T>(stopper).then((result) => ({
-        ...result,
-        stop: true,
-      })),
-    ]);
-  }
-
-  /**
-   * Fires a batch of test commands until a stop signal is received
-   * @param client - The Redis client to use
-   * @param stopSignalPromise - Promise that resolves when the execution should stop
-   * @param options - Options for the command execution
-   * @returns An object containing the promises of all executed commands and the result of the stop signal
-   */
-  static async fireCommandsUntilStopSignal(
-    client: ReturnType<typeof createClient<any, any, any, any>>,
-    stopSignalPromise: Promise<unknown>,
-    options?: Partial<FireCommandsUntilStopSignalOptions>
-  ) {
-    const executeOptions = {
-      ...TestCommandRunner.defaultOptions,
-      ...options,
-    };
-
-    const commandPromises = [];
-
-    while (true) {
-      for (let i = 0; i < executeOptions.batchSize; i++) {
-        for (const command of executeOptions.createCommands(client)) {
-          commandPromises.push(TestCommandRunner.#toSettled(command()));
-        }
-      }
-
-      const result = await TestCommandRunner.#racePromises({
-        timeout: setTimeout(executeOptions.timeoutMs),
-        stopper: stopSignalPromise,
-      });
-
-      if (result.stop) {
-        return {
-          commandPromises,
-          stopResult: result,
-        };
-      }
-    }
-  }
-}

packages/client/lib/tests/test-scenario/test-scenario.util.ts

@@ -110,8 +110,18 @@ export function getDatabaseConfig(
   };
 }
 
-// TODO this should be moved in the tests utils package
+/**
-export async function blockSetImmediate(fn: () => Promise<unknown>) {
+ * Executes the provided function in a context where setImmediate is stubbed to not do anything.
+ * This blocks setImmediate callbacks from executing
+ *
+ * @param command - The command to execute
+ * @returns The error and duration of the command execution
+ */
+export async function blockCommand(command: () => Promise<unknown>) {
+  let error: any;
+
+  const start = performance.now();
+
   let setImmediateStub: any;
 
   try {
@@ -119,79 +129,46 @@ export async function blockSetImmediate(fn: () => Promise<unknown>) {
     setImmediateStub.callsFake(() => {
       //Dont call the callback, effectively blocking execution
     });
-    await fn();
+    await command();
+  } catch (err: any) {
+    error = err;
   } finally {
     if (setImmediateStub) {
       setImmediateStub.restore();
     }
   }
+
+  return {
+    error,
+    duration: performance.now() - start,
+  };
 }
 
 /**
- * Factory class for creating and managing Redis clients
+ * Creates a test client with the provided configuration, connects it and attaches an error handler listener
+ * @param clientConfig - The Redis connection configuration
+ * @param options - Optional client options
+ * @returns The created Redis client
  */
-export class ClientFactory {
+export async function createTestClient(
-  private readonly clients = new Map<
+  clientConfig: RedisConnectionConfig,
-    string,
+  options: Partial<RedisClientOptions> = {}
-    ReturnType<typeof createClient<any, any, any, any>>
+) {
-  >();
+  const client = createClient({
+    socket: {
+      host: clientConfig.host,
+      port: clientConfig.port,
+      ...(clientConfig.tls === true ? { tls: true } : {}),
+    },
+    password: clientConfig.password,
+    username: clientConfig.username,
+    RESP: 3,
+    maintPushNotifications: "auto",
+    maintMovingEndpointType: "auto",
+    ...options,
+  });
 
-  constructor(private readonly config: RedisConnectionConfig) {}
+  await client.connect();
 
-  /**
+  return client;
-   * Creates a new client with the specified options and connects it to the database
-   * @param key - The key to store the client under
-   * @param options - Optional client options
-   * @returns The created and connected client
-   */
-  async create(key: string, options: Partial<RedisClientOptions> = {}) {
-    const client = createClient({
-      socket: {
-        host: this.config.host,
-        port: this.config.port,
-        ...(this.config.tls === true ? { tls: true } : {}),
-      },
-      password: this.config.password,
-      username: this.config.username,
-      RESP: 3,
-      maintPushNotifications: "auto",
-      maintMovingEndpointType: "auto",
-      ...options,
-    });
-
-    client.on("error", (err: Error) => {
-      throw new Error(`Client error: ${err.message}`);
-    });
-
-    await client.connect();
-
-    this.clients.set(key, client);
-
-    return client;
-  }
-
-  /**
-   * Gets an existing client by key or the first one if no key is provided
-   * @param key - The key of the client to retrieve
-   * @returns The client if found, undefined otherwise
-   */
-  get(key?: string) {
-    if (key) {
-      return this.clients.get(key);
-    }
-
-    // Get the first one if no key is provided
-    return this.clients.values().next().value;
-  }
-
-  /**
-   * Destroys all created clients
-   */
-  destroyAll() {
-    this.clients.forEach((client) => {
-      if (client && client.isOpen) {
-        client.destroy();
-      }
-    });
-  }
 }

packages/client/lib/tests/test-scenario/timeout-during-notifications.e2e.ts

@@ -2,22 +2,48 @@ import assert from "node:assert";
 
 import { FaultInjectorClient } from "./fault-injector-client";
 import {
-  ClientFactory,
   getDatabaseConfig,
   getDatabaseConfigFromEnv,
   getEnvConfig,
   RedisConnectionConfig,
-  blockSetImmediate
+  blockCommand,
+  createTestClient,
 } from "./test-scenario.util";
 import { createClient } from "../../..";
 import { before } from "mocha";
-import { TestCommandRunner } from "./test-command-runner";
+import diagnostics_channel from "node:diagnostics_channel";
+import { DiagnosticsEvent } from "../../client/enterprise-maintenance-manager";
 
 describe("Timeout Handling During Notifications", () => {
   let clientConfig: RedisConnectionConfig;
-  let clientFactory: ClientFactory;
   let faultInjectorClient: FaultInjectorClient;
-  let defaultClient: ReturnType<typeof createClient<any, any, any, any>>;
+  let client: ReturnType<typeof createClient<any, any, any, any>>;
+
+  const NORMAL_COMMAND_TIMEOUT = 50;
+  const RELAXED_COMMAND_TIMEOUT = 2000;
+
+  /**
+   * Creates a handler for the `redis.maintenance` channel that will execute and block a command on the client
+   * when a notification is received and save the result in the `result` object.
+   * This is used to test that the command timeout is relaxed during notifications.
+   */
+  const createNotificationMessageHandler = (
+    client: ReturnType<typeof createClient<any, any, any, any>>,
+    result: Record<DiagnosticsEvent["type"], { error: any; duration: number }>,
+    notifications: Array<DiagnosticsEvent["type"]>
+  ) => {
+    return (message: unknown) => {
+      if (notifications.includes((message as DiagnosticsEvent).type)) {
+        setImmediate(async () => {
+          result[(message as DiagnosticsEvent).type] = await blockCommand(
+            async () => {
+              await client.set("key", "value");
+            }
+          );
+        });
+      }
+    };
+  };
 
   before(() => {
     const envConfig = getEnvConfig();
@@ -27,96 +53,64 @@ describe("Timeout Handling During Notifications", () => {
 
     clientConfig = getDatabaseConfig(redisConfig);
     faultInjectorClient = new FaultInjectorClient(envConfig.faultInjectorUrl);
-    clientFactory = new ClientFactory(clientConfig);
   });
 
   beforeEach(async () => {
-    defaultClient = await clientFactory.create("default");
+    client = await createTestClient(clientConfig, {
-
+      commandOptions: { timeout: NORMAL_COMMAND_TIMEOUT },
-    await defaultClient.flushAll();
+      maintRelaxedCommandTimeout: RELAXED_COMMAND_TIMEOUT,
-  });
-
-  afterEach(async () => {
-    clientFactory.destroyAll();
-  });
-
-  it("should relax command timeout on MOVING, MIGRATING, and MIGRATED", async () => {
-    // PART 1
-    // Set very low timeout to trigger errors
-    const lowTimeoutClient = await clientFactory.create("lowTimeout", {
-      maintRelaxedCommandTimeout: 50,
     });
 
-    const { action_id: lowTimeoutBindAndMigrateActionId } =
+    await client.flushAll();
-      await faultInjectorClient.migrateAndBindAction({
+  });
-        bdbId: clientConfig.bdbId,
-        clusterIndex: 0,
-      });
 
-    const lowTimeoutWaitPromise = faultInjectorClient.waitForAction(
+  afterEach(() => {
-      lowTimeoutBindAndMigrateActionId
+    if (client && client.isOpen) {
+      client.destroy();
+    }
+  });
+
+  it("should relax command timeout on MOVING, MIGRATING", async () => {
+    // PART 1
+    // Normal command timeout
+    const { error, duration } = await blockCommand(async () => {
+      await client.set("key", "value");
+    });
+
+    assert.ok(
+      error instanceof Error,
+      "Command Timeout error should be instanceof Error"
+    );
+    assert.ok(
+      duration > NORMAL_COMMAND_TIMEOUT &&
+        duration < NORMAL_COMMAND_TIMEOUT * 1.1,
+      `Normal command should timeout within normal timeout ms`
     );
-
-    const lowTimeoutCommandPromises =
-      await TestCommandRunner.fireCommandsUntilStopSignal(
-        lowTimeoutClient,
-        lowTimeoutWaitPromise
-      );
-
-    const lowTimeoutRejectedCommands = (
-      await Promise.all(lowTimeoutCommandPromises.commandPromises)
-    ).filter((result) => result.status === "rejected");
-
-    assert.ok(lowTimeoutRejectedCommands.length > 0);
     assert.strictEqual(
-      lowTimeoutRejectedCommands.filter((rejected) => {
+      error?.constructor?.name,
-        return (
+      "TimeoutError",
-          // TODO instanceof doesn't work for some reason
+      "Command Timeout error should be TimeoutError"
-          rejected.error.constructor.name ===
-          "CommandTimeoutDuringMaintananceError"
-        );
-      }).length,
-      lowTimeoutRejectedCommands.length
     );
 
     // PART 2
-    // Set high timeout to avoid errors
+    // Command timeout during maintenance
-    const highTimeoutClient = await clientFactory.create("highTimeout", {
+    const notifications: Array<DiagnosticsEvent["type"]> = [
-      maintRelaxedCommandTimeout: 10000,
+      "MOVING",
-    });
+      "MIGRATING",
+    ];
 
-    const { action_id: highTimeoutBindAndMigrateActionId } =
+    const result: Record<
-      await faultInjectorClient.migrateAndBindAction({
+      DiagnosticsEvent["type"],
-        bdbId: clientConfig.bdbId,
+      { error: any; duration: number }
-        clusterIndex: 0,
+    > = {};
-      });
 
-    const highTimeoutWaitPromise = faultInjectorClient.waitForAction(
+    const onMessageHandler = createNotificationMessageHandler(
-      highTimeoutBindAndMigrateActionId
+      client,
+      result,
+      notifications
     );
 
-    const highTimeoutCommandPromises =
+    diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
-      await TestCommandRunner.fireCommandsUntilStopSignal(
-        highTimeoutClient,
-        highTimeoutWaitPromise
-      );
-
-    const highTimeoutRejectedCommands = (
-      await Promise.all(highTimeoutCommandPromises.commandPromises)
-    ).filter((result) => result.status === "rejected");
-
-    assert.strictEqual(highTimeoutRejectedCommands.length, 0);
-  });
-
-  it("should unrelax command timeout after MAINTENANCE", async () => {
-    const clientWithCommandTimeout = await clientFactory.create(
-      "clientWithCommandTimeout",
-      {
-        commandOptions: {
-          timeout: 100,
-        },
-      }
-    );
 
     const { action_id: bindAndMigrateActionId } =
       await faultInjectorClient.migrateAndBindAction({
@@ -124,36 +118,173 @@ describe("Timeout Handling During Notifications", () => {
         clusterIndex: 0,
       });
 
-    const lowTimeoutWaitPromise = faultInjectorClient.waitForAction(
+    await faultInjectorClient.waitForAction(bindAndMigrateActionId);
-      bindAndMigrateActionId
+
+    diagnostics_channel.unsubscribe("redis.maintenance", onMessageHandler);
+
+    notifications.forEach((notification) => {
+      assert.ok(
+        result[notification]?.error instanceof Error,
+        `${notification} notification error should be instanceof Error`
+      );
+      assert.ok(
+        result[notification]?.duration > RELAXED_COMMAND_TIMEOUT &&
+          result[notification]?.duration < RELAXED_COMMAND_TIMEOUT * 1.1,
+        `${notification} notification should timeout within relaxed timeout`
+      );
+      assert.strictEqual(
+        result[notification]?.error?.constructor?.name,
+        "CommandTimeoutDuringMaintenanceError",
+        `${notification} notification error should be CommandTimeoutDuringMaintenanceError`
+      );
+    });
+  });
+
+  it("should unrelax command timeout after MIGRATED and  MOVING", async () => {
+    const { action_id: migrateActionId } =
+      await faultInjectorClient.triggerAction({
+        type: "migrate",
+        parameters: {
+          cluster_index: 0,
+        },
+      });
+
+    await faultInjectorClient.waitForAction(migrateActionId);
+
+    // PART 1
+    // After migration
+    const { error: errorMigrate, duration: durationMigrate } =
+      await blockCommand(async () => {
+        await client.set("key", "value");
+      });
+
+    assert.ok(
+      errorMigrate instanceof Error,
+      "Command Timeout error should be instanceof Error"
+    );
+    assert.ok(
+      durationMigrate > NORMAL_COMMAND_TIMEOUT &&
+        durationMigrate < NORMAL_COMMAND_TIMEOUT * 1.1,
+      `Normal command should timeout within normal timeout ms`
+    );
+    assert.strictEqual(
+      errorMigrate?.constructor?.name,
+      "TimeoutError",
+      "Command Timeout error should be TimeoutError"
     );
 
-    const relaxedTimeoutCommandPromises =
+    const { action_id: bindActionId } = await faultInjectorClient.triggerAction(
-      await TestCommandRunner.fireCommandsUntilStopSignal(
+      {
-        clientWithCommandTimeout,
+        type: "bind",
-        lowTimeoutWaitPromise
+        parameters: {
-      );
+          bdb_id: clientConfig.bdbId.toString(),
-
+          cluster_index: 0,
-    const relaxedTimeoutRejectedCommands = (
+        },
-      await Promise.all(relaxedTimeoutCommandPromises.commandPromises)
-    ).filter((result) => result.status === "rejected");
-
-    assert.ok(relaxedTimeoutRejectedCommands.length === 0);
-
-    const start = performance.now();
-
-    let error: any;
-    await blockSetImmediate(async () => {
-      try {
-        await clientWithCommandTimeout.set("key", "value");
-      } catch (err: any) {
-        error = err;
       }
+    );
+
+    await faultInjectorClient.waitForAction(bindActionId);
+
+    // PART 2
+    // After bind
+    const { error: errorBind, duration: durationBind } = await blockCommand(
+      async () => {
+        await client.set("key", "value");
+      }
+    );
+
+    assert.ok(
+      errorBind instanceof Error,
+      "Command Timeout error should be instanceof Error"
+    );
+    assert.ok(
+      durationBind > NORMAL_COMMAND_TIMEOUT &&
+        durationBind < NORMAL_COMMAND_TIMEOUT * 1.1,
+      `Normal command should timeout within normal timeout ms`
+    );
+    assert.strictEqual(
+      errorBind?.constructor?.name,
+      "TimeoutError",
+      "Command Timeout error should be TimeoutError"
+    );
+  });
+
+  it("should relax command timeout on FAILING_OVER", async () => {
+    const notifications: Array<DiagnosticsEvent["type"]> = ["FAILING_OVER"];
+
+    const result: Record<
+      DiagnosticsEvent["type"],
+      { error: any; duration: number }
+    > = {};
+
+    const onMessageHandler = createNotificationMessageHandler(
+      client,
+      result,
+      notifications
+    );
+
+    diagnostics_channel.subscribe("redis.maintenance", onMessageHandler);
+
+    const { action_id: failoverActionId } =
+      await faultInjectorClient.triggerAction({
+        type: "failover",
+        parameters: {
+          bdb_id: clientConfig.bdbId.toString(),
+          cluster_index: 0,
+        },
+      });
+
+    await faultInjectorClient.waitForAction(failoverActionId);
+
+    diagnostics_channel.unsubscribe("redis.maintenance", onMessageHandler);
+
+    notifications.forEach((notification) => {
+      assert.ok(
+        result[notification]?.error instanceof Error,
+        `${notification} notification error should be instanceof Error`
+      );
+      assert.ok(
+        result[notification]?.duration > RELAXED_COMMAND_TIMEOUT &&
+          result[notification]?.duration < RELAXED_COMMAND_TIMEOUT * 1.1,
+        `${notification} notification should timeout within relaxed timeout`
+      );
+      assert.strictEqual(
+        result[notification]?.error?.constructor?.name,
+        "CommandTimeoutDuringMaintenanceError",
+        `${notification} notification error should be CommandTimeoutDuringMaintenanceError`
+      );
+    });
+  });
+
+  it("should unrelax command timeout after FAILED_OVER", async () => {
+    const { action_id: failoverActionId } =
+      await faultInjectorClient.triggerAction({
+        type: "failover",
+        parameters: {
+          bdb_id: clientConfig.bdbId.toString(),
+          cluster_index: 0,
+        },
+      });
+
+    await faultInjectorClient.waitForAction(failoverActionId);
+
+    const { error, duration } = await blockCommand(async () => {
+      await client.set("key", "value");
     });
 
-    // Make sure it took less than 1sec to fail
+    assert.ok(
-    assert.ok(performance.now() - start < 1000);
+      error instanceof Error,
-    assert.ok(error instanceof Error);
+      "Command Timeout error should be instanceof Error"
-    assert.ok(error.constructor.name === "TimeoutError");
+    );
+    assert.ok(
+      duration > NORMAL_COMMAND_TIMEOUT &&
+        duration < NORMAL_COMMAND_TIMEOUT * 1.1,
+      `Normal command should timeout within normal timeout ms`
+    );
+    assert.strictEqual(
+      error?.constructor?.name,
+      "TimeoutError",
+      "Command Timeout error should be TimeoutError"
+    );
   });
 });

packages/client/tsconfig.json

@@ -11,7 +11,8 @@
   "exclude": [
     "./lib/test-utils.ts",
     "./lib/**/*.spec.ts",
-    "./lib/sentinel/test-util.ts"
+    "./lib/sentinel/test-util.ts",
+    "./lib/tests/**/*.ts"
   ],
   "typedocOptions": {
     "entryPoints": [