1
0
mirror of https://github.com/redis/node-redis.git synced 2025-08-04 15:02:09 +03:00

Add support for T-Digest (#2214)

* wip

* close #2216 - add support for TDIGEST.MERGESTORE and make compression optional on TDIGEST.CREATE

* fix some tdigest commands, use bloom edge docker

* fix index.ts

* 2.4-RC2 (v2.4.1)

* fix some commands and tests

* clean code
This commit is contained in:
Leibale Eidelman
2022-11-01 15:45:47 -04:00
committed by GitHub
parent 1c6d74ffcb
commit be90e62360
33 changed files with 794 additions and 20 deletions

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './ADD';
describe('TDIGEST.ADD', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.ADD', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.add', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.add('key', [1])
]);
assert.equal(reply, 'OK');
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,17 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export function transformArguments(
key: RedisCommandArgument,
values: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.ADD', key];
for (const item of values) {
args.push(item.toString());
}
return args;
}
export declare function transformReply(): 'OK';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './BYRANK';
describe('TDIGEST.BYRANK', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.BYRANK', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.byRank', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.byRank('key', [1])
]);
assert.deepEqual(reply, [NaN]);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,19 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
ranks: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.BYRANK', key];
for (const rank of ranks) {
args.push(rank.toString());
}
return args;
}
export { transformDoublesReply as transformReply } from '.';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './BYREVRANK';
describe('TDIGEST.BYREVRANK', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.BYREVRANK', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.byRevRank', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.byRevRank('key', [1])
]);
assert.deepEqual(reply, [NaN]);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,19 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
ranks: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.BYREVRANK', key];
for (const rank of ranks) {
args.push(rank.toString());
}
return args;
}
export { transformDoublesReply as transformReply } from '.';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './CDF';
describe('TDIGEST.CDF', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.CDF', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.cdf', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.cdf('key', [1])
]);
assert.deepEqual(reply, [NaN]);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,19 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
values: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.CDF', key];
for (const item of values) {
args.push(item.toString());
}
return args;
}
export { transformDoublesReply as transformReply } from '.';

View File

@@ -0,0 +1,30 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './CREATE';
describe('TDIGEST.CREATE', () => {
describe('transformArguments', () => {
it('without options', () => {
assert.deepEqual(
transformArguments('key'),
['TDIGEST.CREATE', 'key']
);
});
it('with COMPRESSION', () => {
assert.deepEqual(
transformArguments('key', {
COMPRESSION: 100
}),
['TDIGEST.CREATE', 'key', 'COMPRESSION', '100']
);
});
});
testUtils.testWithClient('client.tDigest.create', async client => {
assert.equal(
await client.tDigest.create('key'),
'OK'
);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,16 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
import { CompressionOption, pushCompressionArgument } from '.';
export const FIRST_KEY_INDEX = 1;
export function transformArguments(
key: RedisCommandArgument,
options?: CompressionOption
): RedisCommandArguments {
return pushCompressionArgument(
['TDIGEST.CREATE', key],
options
);
}
export declare function transformReply(): 'OK';

View File

@@ -0,0 +1,25 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './INFO';
describe('TDIGEST.INFO', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key'),
['TDIGEST.INFO', 'key']
);
});
testUtils.testWithClient('client.tDigest.info', async client => {
await client.tDigest.create('key');
const info = await client.tDigest.info('key');
assert(typeof info.capacity, 'number');
assert(typeof info.mergedNodes, 'number');
assert(typeof info.unmergedNodes, 'number');
assert(typeof info.mergedWeight, 'number');
assert(typeof info.unmergedWeight, 'number');
assert(typeof info.totalCompression, 'number');
assert(typeof info.totalCompression, 'number');
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,51 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
return [
'TDIGEST.INFO',
key
];
}
type InfoRawReply = [
'Compression',
number,
'Capacity',
number,
'Merged nodes',
number,
'Unmerged nodes',
number,
'Merged weight',
string,
'Unmerged weight',
string,
'Total compressions',
number
];
interface InfoReply {
comperssion: number;
capacity: number;
mergedNodes: number;
unmergedNodes: number;
mergedWeight: number;
unmergedWeight: number;
totalCompression: number;
}
export function transformReply(reply: InfoRawReply): InfoReply {
return {
comperssion: reply[1],
capacity: reply[3],
mergedNodes: reply[5],
unmergedNodes: reply[7],
mergedWeight: Number(reply[9]),
unmergedWeight: Number(reply[11]),
totalCompression: reply[13]
};
}

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments, transformReply } from './MAX';
describe('TDIGEST.MAX', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key'),
['TDIGEST.MAX', 'key']
);
});
testUtils.testWithClient('client.tDigest.max', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.max('key')
]);
assert.deepEqual(reply, NaN);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,14 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
return [
'TDIGEST.MAX',
key
];
}
export { transformDoubleReply as transformReply } from '.';

View File

@@ -0,0 +1,50 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments, transformReply } from './MERGE';
describe('TDIGEST.MERGE', () => {
describe('transformArguments', () => {
describe('srcKeys', () => {
it('string', () => {
assert.deepEqual(
transformArguments('dest', 'src'),
['TDIGEST.MERGE', 'dest', '1', 'src']
);
});
it('Array', () => {
assert.deepEqual(
transformArguments('dest', ['1', '2']),
['TDIGEST.MERGE', 'dest', '2', '1', '2']
);
});
});
it('with COMPRESSION', () => {
assert.deepEqual(
transformArguments('dest', 'src', {
COMPRESSION: 100
}),
['TDIGEST.MERGE', 'dest', '1', 'src', 'COMPRESSION', '100']
);
});
it('with OVERRIDE', () => {
assert.deepEqual(
transformArguments('dest', 'src', {
OVERRIDE: true
}),
['TDIGEST.MERGE', 'dest', '1', 'src', 'OVERRIDE']
);
});
});
testUtils.testWithClient('client.tDigest.merge', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('src'),
client.tDigest.merge('dest', 'src')
]);
assert.equal(reply, 'OK');
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,30 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
import { pushVerdictArgument } from '@redis/client/dist/lib/commands/generic-transformers';
import { CompressionOption, pushCompressionArgument } from '.';
export const FIRST_KEY_INDEX = 1;
interface MergeOptions extends CompressionOption {
OVERRIDE?: boolean;
}
export function transformArguments(
destKey: RedisCommandArgument,
srcKeys: RedisCommandArgument | Array<RedisCommandArgument>,
options?: MergeOptions
): RedisCommandArguments {
const args = pushVerdictArgument(
['TDIGEST.MERGE', destKey],
srcKeys
);
pushCompressionArgument(args, options);
if (options?.OVERRIDE) {
args.push('OVERRIDE');
}
return args;
}
export declare function transformReply(): 'OK';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments, transformReply } from './MIN';
describe('TDIGEST.MIN', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key'),
['TDIGEST.MIN', 'key']
);
});
testUtils.testWithClient('client.tDigest.min', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.min('key')
]);
assert.equal(reply, NaN);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,14 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
return [
'TDIGEST.MIN',
key
];
}
export { transformDoubleReply as transformReply } from '.';

View File

@@ -0,0 +1,24 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './QUANTILE';
describe('TDIGEST.QUANTILE', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.QUANTILE', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.quantile', async client => {
const [, reply] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.quantile('key', [1])
]);
assert.deepEqual(
reply,
[NaN]
);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,23 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
quantiles: Array<number>
): RedisCommandArguments {
const args = [
'TDIGEST.QUANTILE',
key
];
for (const quantile of quantiles) {
args.push(quantile.toString());
}
return args;
}
export { transformDoublesReply as transformReply } from '.';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './RANK';
describe('TDIGEST.RANK', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.RANK', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.rank', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.rank('key', [1])
]);
assert.deepEqual(reply, [-2]);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,19 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
values: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.RANK', key];
for (const item of values) {
args.push(item.toString());
}
return args;
}
export declare function transformReply(): Array<number>;

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './RESET';
describe('TDIGEST.RESET', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key'),
['TDIGEST.RESET', 'key']
);
});
testUtils.testWithClient('client.tDigest.reset', async client => {
const [, reply] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.reset('key')
]);
assert.equal(reply, 'OK');
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,9 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export function transformArguments(key: RedisCommandArgument): RedisCommandArguments {
return ['TDIGEST.RESET', key];
}
export declare function transformReply(): 'OK';

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments } from './REVRANK';
describe('TDIGEST.REVRANK', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', [1, 2]),
['TDIGEST.REVRANK', 'key', '1', '2']
);
});
testUtils.testWithClient('client.tDigest.revRank', async client => {
const [ , reply ] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.revRank('key', [1])
]);
assert.deepEqual(reply, [-2]);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,19 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
values: Array<number>
): RedisCommandArguments {
const args = ['TDIGEST.REVRANK', key];
for (const item of values) {
args.push(item.toString());
}
return args;
}
export declare function transformReply(): Array<number>;

View File

@@ -0,0 +1,21 @@
import { strict as assert } from 'assert';
import testUtils, { GLOBAL } from '../../test-utils';
import { transformArguments, transformReply } from './TRIMMED_MEAN';
describe('TDIGEST.RESET', () => {
it('transformArguments', () => {
assert.deepEqual(
transformArguments('key', 0, 1),
['TDIGEST.TRIMMED_MEAN', 'key', '0', '1']
);
});
testUtils.testWithClient('client.tDigest.trimmedMean', async client => {
const [, reply] = await Promise.all([
client.tDigest.create('key'),
client.tDigest.trimmedMean('key', 0, 1)
]);
assert.equal(reply, NaN);
}, GLOBAL.SERVERS.OPEN);
});

View File

@@ -0,0 +1,20 @@
import { RedisCommandArgument, RedisCommandArguments } from '@redis/client/dist/lib/commands';
export const FIRST_KEY_INDEX = 1;
export const IS_READ_ONLY = true;
export function transformArguments(
key: RedisCommandArgument,
lowCutPercentile: number,
highCutPercentile: number
): RedisCommandArguments {
return [
'TDIGEST.TRIMMED_MEAN',
key,
lowCutPercentile.toString(),
highCutPercentile.toString()
];
}
export { transformDoubleReply as transformReply } from '.';

View File

@@ -0,0 +1,55 @@
import { strict as assert } from 'assert';
import { pushCompressionArgument, transformDoubleReply, transformDoublesReply } from '.';
describe('pushCompressionArgument', () => {
it('undefined', () => {
assert.deepEqual(
pushCompressionArgument([]),
[]
);
});
it('100', () => {
assert.deepEqual(
pushCompressionArgument([], { COMPRESSION: 100 }),
['COMPRESSION', '100']
);
});
});
describe('transformDoubleReply', () => {
it('inf', () => {
assert.equal(
transformDoubleReply('inf'),
Infinity
);
});
it('-inf', () => {
assert.equal(
transformDoubleReply('-inf'),
-Infinity
);
});
it('nan', () => {
assert.equal(
transformDoubleReply('nan'),
NaN
);
});
it('0', () => {
assert.equal(
transformDoubleReply('0'),
0
);
});
});
it('transformDoublesReply', () => {
assert.deepEqual(
transformDoublesReply(['inf', '-inf', 'nan', '0']),
[Infinity, -Infinity, NaN, 0]
);
});

View File

@@ -0,0 +1,81 @@
import { RedisCommandArguments } from '@redis/client/dist/lib/commands';
import * as ADD from './ADD';
import * as BYRANK from './BYRANK';
import * as BYREVRANK from './BYREVRANK';
import * as CDF from './CDF';
import * as CREATE from './CREATE';
import * as INFO from './INFO';
import * as MAX from './MAX';
import * as MERGE from './MERGE';
import * as MIN from './MIN';
import * as QUANTILE from './QUANTILE';
import * as RANK from './RANK';
import * as RESET from './RESET';
import * as REVRANK from './REVRANK';
import * as TRIMMED_MEAN from './TRIMMED_MEAN';
export default {
ADD,
add: ADD,
BYRANK,
byRank: BYRANK,
BYREVRANK,
byRevRank: BYREVRANK,
CDF,
cdf: CDF,
CREATE,
create: CREATE,
INFO,
info: INFO,
MAX,
max: MAX,
MERGE,
merge: MERGE,
MIN,
min: MIN,
QUANTILE,
quantile: QUANTILE,
RANK,
rank: RANK,
RESET,
reset: RESET,
REVRANK,
revRank: REVRANK,
TRIMMED_MEAN,
trimmedMean: TRIMMED_MEAN
};
export interface CompressionOption {
COMPRESSION?: number;
}
export function pushCompressionArgument(
args: RedisCommandArguments,
options?: CompressionOption
): RedisCommandArguments {
if (options?.COMPRESSION) {
args.push('COMPRESSION', options.COMPRESSION.toString());
}
return args;
}
export function transformDoubleReply(reply: string): number {
switch (reply) {
case 'inf':
return Infinity;
case '-inf':
return -Infinity;
case 'nan':
return NaN;
default:
return parseFloat(reply);
}
}
export function transformDoublesReply(reply: Array<string>): Array<number> {
return reply.map(transformDoubleReply);
}