Add support for sharded PubSub (#2373)

* refactor pubsub, add support for sharded pub sub * run tests in redis 7 only, fix PUBSUB SHARDCHANNELS test * add some comments and fix some bugs * PubSubType, not PubSubTypes 🤦‍♂️ * remove test.txt * fix some bugs, add tests * add some tests * fix #2345 - allow PING in PubSub mode (remove client side validation) * remove .only * revert changes in cluster/index.ts * fix tests minimum version * handle server sunsubscribe * add 'sharded-channel-moved' event to docs, improve the events section in the main README (fix #2302) * exit "resubscribe" if pubsub not active * Update commands-queue.ts * Release client@1.5.0-rc.0 * WIP * use `node:util` instead of `node:util/types` (to support node 14) * run PubSub resharding test with Redis 7+ * fix inconsistency in live resharding test * add some tests * fix iterateAllNodes when starting from a replica * fix iterateAllNodes random * fix slotNodesIterator * fix slotNodesIterator * clear pubSubNode when node in use * wait for all nodes cluster state to be ok before testing * `cluster.minimizeConections` tests * `client.reconnectStrategry = false | 0` tests * sharded pubsub + cluster 🎉 * add minimum version to sharded pubsub tests * add cluster sharded pubsub live reshard test, use stable dockers for tests, make sure to close pubsub clients when a node disconnects from the cluster * fix "ssubscribe & sunsubscribe" test * lock search docker to 2.4.9 * change numberOfMasters default to 2 * use edge for bloom * add tests * add back getMasters and getSlotMaster as deprecated functions * add some tests * fix reconnect strategy + docs * sharded pubsub docs * Update pub-sub.md * some jsdoc, docs, cluster topology test * clean pub-sub docs Co-authored-by: Simon Prickett <simon@redislabs.com> * reconnect startegy docs and bug fix Co-authored-by: Simon Prickett <simon@redislabs.com> * refine jsdoc and some docs Co-authored-by: Simon Prickett <simon@redislabs.com> * I'm stupid * fix cluster topology test * fix cluster topology test * Update README.md * Update clustering.md * Update pub-sub.md Co-authored-by: Simon Prickett <simon@redislabs.com>
2025-08-06 02:15:48 +03:00 · 2023-01-25 11:00:39 -05:00
parent e75a5db3e4
commit 3b1bad2296
41 changed files with 2415 additions and 768 deletions
--- a/packages/client/lib/commands/CLUSTER_BUMPEPOCH.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_BUMPEPOCH.spec.ts
@@ -11,8 +11,9 @@ describe('CLUSTER BUMPEPOCH', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterBumpEpoch', async cluster => {
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.equal(
-            typeof await cluster.getSlotMaster(0).client.clusterBumpEpoch(),
+            typeof await client.clusterBumpEpoch(),
            'string'
        );
    }, GLOBAL.SERVERS.OPEN);
--- a/packages/client/lib/commands/CLUSTER_COUNT-FAILURE-REPORTS.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_COUNT-FAILURE-REPORTS.spec.ts
@@ -11,7 +11,7 @@ describe('CLUSTER COUNT-FAILURE-REPORTS', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterCountFailureReports', async cluster => {
-        const { client } = cluster.getSlotMaster(0);
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.equal(
            typeof await client.clusterCountFailureReports(
                await client.clusterMyId()
--- a/packages/client/lib/commands/CLUSTER_COUNTKEYSINSLOT.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_COUNTKEYSINSLOT.spec.ts
@@ -11,8 +11,9 @@ describe('CLUSTER COUNTKEYSINSLOT', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterCountKeysInSlot', async cluster => {
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.equal(
-            typeof await cluster.getSlotMaster(0).client.clusterCountKeysInSlot(0),
+            typeof await client.clusterCountKeysInSlot(0),
            'number'
        );
    }, GLOBAL.CLUSTERS.OPEN);
--- a/packages/client/lib/commands/CLUSTER_GETKEYSINSLOT.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_GETKEYSINSLOT.spec.ts
@@ -11,7 +11,8 @@ describe('CLUSTER GETKEYSINSLOT', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterGetKeysInSlot', async cluster => {
-        const reply = await cluster.getSlotMaster(0).client.clusterGetKeysInSlot(0, 1);
+        const client = await cluster.nodeClient(cluster.masters[0]),
+            reply = await client.clusterGetKeysInSlot(0, 1);
        assert.ok(Array.isArray(reply));
        for (const item of reply) {
            assert.equal(typeof item, 'string');
--- a/packages/client/lib/commands/CLUSTER_INFO.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_INFO.spec.ts
@@ -46,8 +46,9 @@ describe('CLUSTER INFO', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterInfo', async cluster => {
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.notEqual(
-            await cluster.getSlotMaster(0).client.clusterInfo(),
+            await client.clusterInfo(),
            null
        );
    }, GLOBAL.CLUSTERS.OPEN);
--- a/packages/client/lib/commands/CLUSTER_KEYSLOT.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_KEYSLOT.spec.ts
@@ -11,8 +11,9 @@ describe('CLUSTER KEYSLOT', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterKeySlot', async cluster => {
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.equal(
-            typeof await cluster.getSlotMaster(0).client.clusterKeySlot('key'),
+            typeof await client.clusterKeySlot('key'),
            'number'
        );
    }, GLOBAL.CLUSTERS.OPEN);
--- a/packages/client/lib/commands/CLUSTER_LINKS.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_LINKS.spec.ts
@@ -13,7 +13,8 @@ describe('CLUSTER LINKS', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterLinks', async cluster => {
-        const links = await cluster.getSlotMaster(0).client.clusterLinks();
+        const client = await cluster.nodeClient(cluster.masters[0]),
+            links = await client.clusterLinks();
        assert.ok(Array.isArray(links));
        for (const link of links) {
            assert.equal(typeof link.direction, 'string');
--- a/packages/client/lib/commands/CLUSTER_MYID.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_MYID.spec.ts
@@ -11,9 +11,11 @@ describe('CLUSTER MYID', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterMyId', async cluster => {
+        const [master] = cluster.masters,
+            client = await cluster.nodeClient(master);
        assert.equal(
-            typeof await cluster.getSlotMaster(0).client.clusterMyId(),
-            'string'
+            await client.clusterMyId(),
+            master.id
        );
    }, GLOBAL.CLUSTERS.OPEN);
 });
--- a/packages/client/lib/commands/CLUSTER_SAVECONFIG.spec.ts
+++ b/packages/client/lib/commands/CLUSTER_SAVECONFIG.spec.ts
@@ -11,8 +11,9 @@ describe('CLUSTER SAVECONFIG', () => {
    });

    testUtils.testWithCluster('clusterNode.clusterSaveConfig', async cluster => {
+        const client = await cluster.nodeClient(cluster.masters[0]);
        assert.equal(
-            await cluster.getSlotMaster(0).client.clusterSaveConfig(),
+            await client.clusterSaveConfig(),
            'OK'
        );
    }, GLOBAL.CLUSTERS.OPEN);
--- a/packages/client/lib/commands/CLUSTER_SLOTS.ts
+++ b/packages/client/lib/commands/CLUSTER_SLOTS.ts
@@ -13,7 +13,7 @@ type ClusterSlotsRawReply = Array<[
    ...replicas: Array<ClusterSlotsRawNode>
 ]>;

-type ClusterSlotsNode = {
+export interface ClusterSlotsNode {
    ip: string;
    port: number;
    id: string;
--- a/packages/client/lib/commands/PING.spec.ts
+++ b/packages/client/lib/commands/PING.spec.ts
@@ -1,8 +1,24 @@
 import { strict as assert } from 'assert';
 import testUtils, { GLOBAL } from '../test-utils';
-import RedisClient from '../client';
+import { transformArguments } from './PING';

 describe('PING', () => {
+    describe('transformArguments', () => {
+        it('default', () => {
+            assert.deepEqual(
+                transformArguments(),
+                ['PING']
+            );
+        });
+
+        it('with message', () => {
+            assert.deepEqual(
+                transformArguments('message'),
+                ['PING', 'message']
+            );
+        });
+    });
+
    describe('client.ping', () => {
        testUtils.testWithClient('string', async client => {
            assert.equal(
@@ -13,7 +29,7 @@ describe('PING', () => {

        testUtils.testWithClient('buffer', async client => {
            assert.deepEqual(
-                await client.ping(RedisClient.commandOptions({ returnBuffers: true })),
+                await client.ping(client.commandOptions({ returnBuffers: true })),
                Buffer.from('PONG')
            );
        }, GLOBAL.SERVERS.OPEN);
--- a/packages/client/lib/commands/PING.ts
+++ b/packages/client/lib/commands/PING.ts
@@ -1,7 +1,12 @@
-import { RedisCommandArgument } from '.';
+import { RedisCommandArgument, RedisCommandArguments } from '.';

-export function transformArguments(): Array<string> {
-    return ['PING'];
+export function transformArguments(message?: RedisCommandArgument): RedisCommandArguments {
+    const args: RedisCommandArguments = ['PING'];
+    if (message) {
+        args.push(message);
+    }
+
+    return args;
 }

 export declare function transformReply(): RedisCommandArgument;
--- a/packages/client/lib/commands/PUBLISH.ts
+++ b/packages/client/lib/commands/PUBLISH.ts
@@ -1,5 +1,7 @@
 import { RedisCommandArgument, RedisCommandArguments } from '.';

+export const IS_READ_ONLY = true;
+
 export function transformArguments(
    channel: RedisCommandArgument,
    message: RedisCommandArgument
--- a/packages/client/lib/commands/PUBSUB_SHARDCHANNELS.spec.ts
+++ b/packages/client/lib/commands/PUBSUB_SHARDCHANNELS.spec.ts
@@ -0,0 +1,30 @@
+import { strict as assert } from 'assert';
+import testUtils, { GLOBAL } from '../test-utils';
+import { transformArguments } from './PUBSUB_SHARDCHANNELS';
+
+describe('PUBSUB SHARDCHANNELS', () => {
+    testUtils.isVersionGreaterThanHook([7]);
+    
+    describe('transformArguments', () => {
+        it('without pattern', () => {
+            assert.deepEqual(
+                transformArguments(),
+                ['PUBSUB', 'SHARDCHANNELS']
+            );
+        });
+
+        it('with pattern', () => {
+            assert.deepEqual(
+                transformArguments('patter*'),
+                ['PUBSUB', 'SHARDCHANNELS', 'patter*']
+            );
+        });
+    });
+
+    testUtils.testWithClient('client.pubSubShardChannels', async client => {
+        assert.deepEqual(
+            await client.pubSubShardChannels(),
+            []
+        );
+    }, GLOBAL.SERVERS.OPEN);
+});
--- a/packages/client/lib/commands/PUBSUB_SHARDCHANNELS.ts
+++ b/packages/client/lib/commands/PUBSUB_SHARDCHANNELS.ts
@@ -0,0 +1,13 @@
+import { RedisCommandArgument, RedisCommandArguments } from '.';
+
+export const IS_READ_ONLY = true;
+
+export function transformArguments(
+    pattern?: RedisCommandArgument
+): RedisCommandArguments {
+    const args: RedisCommandArguments = ['PUBSUB', 'SHARDCHANNELS'];
+    if (pattern) args.push(pattern);
+    return args;
+}
+
+export declare function transformReply(): Array<RedisCommandArgument>;
--- a/packages/client/lib/commands/SPUBLISH.spec.ts
+++ b/packages/client/lib/commands/SPUBLISH.spec.ts
@@ -0,0 +1,21 @@
+import { strict as assert } from 'assert';
+import testUtils, { GLOBAL } from '../test-utils';
+import { transformArguments } from './SPUBLISH';
+
+describe('SPUBLISH', () => {
+    testUtils.isVersionGreaterThanHook([7]);
+    
+    it('transformArguments', () => {
+        assert.deepEqual(
+            transformArguments('channel', 'message'),
+            ['SPUBLISH', 'channel', 'message']
+        );
+    });
+
+    testUtils.testWithClient('client.sPublish', async client => {
+        assert.equal(
+            await client.sPublish('channel', 'message'),
+            0
+        );
+    }, GLOBAL.SERVERS.OPEN);
+});
--- a/packages/client/lib/commands/SPUBLISH.ts
+++ b/packages/client/lib/commands/SPUBLISH.ts
@@ -0,0 +1,14 @@
+import { RedisCommandArgument, RedisCommandArguments } from '.';
+
+export const IS_READ_ONLY = true;
+
+export const FIRST_KEY_INDEX = 1;
+
+export function transformArguments(
+    channel: RedisCommandArgument,
+    message: RedisCommandArgument
+): RedisCommandArguments {
+    return ['SPUBLISH', channel, message];
+}
+
+export declare function transformReply(): number;
--- a/packages/client/lib/commands/generic-transformers.ts
+++ b/packages/client/lib/commands/generic-transformers.ts
@@ -137,7 +137,6 @@ export function transformSortedSetMemberNullReply(
 export function transformSortedSetMemberReply(
    reply: [RedisCommandArgument, RedisCommandArgument]
 ): ZMember {
-
    return {
        value: reply[0],
        score: transformNumberInfinityReply(reply[1])