DOC-4039: add TCEs to the query pages - reissue (#2846)

doctests/package.json

@@ -6,7 +6,8 @@
   "private": true,
   "type": "module",
   "dependencies": {
-    "redis": "../"
+    "redis": "../",
+    "@xenova/transformers": "^2.17.2"
   }
 }
 

doctests/query-agg.js

@@ -0,0 +1,139 @@
+// EXAMPLE: query_agg
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient } from 'redis';
+import { SchemaFieldTypes, AggregateSteps, AggregateGroupByReducers } from '@redis/search';
+
+const client = createClient();
+
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+  '$.condition': {
+    type: SchemaFieldTypes.TAG,
+    AS: 'condition'
+  },
+  '$.price': {
+    type: SchemaFieldTypes.NUMERIC,
+    AS: 'price'
+  }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+})
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_em.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START agg1
+const res1 = await client.ft.aggregate('idx:bicycle', '@condition:{new}', {
+  LOAD: ['__key', 'price'],
+  APPLY: {
+    expression: '@price - (@price * 0.1)',
+    AS: 'discounted'
+  }
+});
+
+console.log(res1.results.length); // >>> 5
+console.log(res1.results); // >>>
+//[
+//  [Object: null prototype] { __key: 'bicycle:0', price: '270' },
+//  [Object: null prototype] { __key: 'bicycle:5', price: '810' },
+//  [Object: null prototype] { __key: 'bicycle:6', price: '2300' },
+//  [Object: null prototype] { __key: 'bicycle:7', price: '430' },
+//  [Object: null prototype] { __key: 'bicycle:8', price: '1200' }
+//]
+// REMOVE_START
+assert.strictEqual(res1.results.length, 5);
+// REMOVE_END
+// STEP_END
+
+// STEP_START agg2
+const res2 = await client.ft.aggregate('idx:bicycle', '*', {
+  LOAD: ['@price'],
+  STEPS: [{
+      type: AggregateSteps.APPLY,
+      expression: '@price<1000',
+      AS: 'price_category'
+    },{
+      type: AggregateSteps.GROUPBY,
+      properties: '@condition',
+      REDUCE:[{
+        type: AggregateGroupByReducers.SUM,
+        property: '@price_category',
+        AS: 'num_affordable'
+      }]
+    }]
+});
+console.log(res2.results.length); // >>> 3
+console.log(res2.results); // >>>
+//[[Object: null prototype] { condition: 'refurbished', num_affordable: '1' },
+//  [Object: null prototype] { condition: 'used', num_affordable: '1' },
+//  [Object: null prototype] { condition: 'new', num_affordable: '3' }
+//]
+// REMOVE_START
+assert.strictEqual(res2.results.length, 3);
+// REMOVE_END
+// STEP_END
+
+// STEP_START agg3
+const res3 = await client.ft.aggregate('idx:bicycle', '*', {
+  STEPS: [{
+      type: AggregateSteps.APPLY,
+      expression: "'bicycle'",
+      AS: 'type'
+    }, {
+      type: AggregateSteps.GROUPBY,
+      properties: '@type',
+      REDUCE: [{
+        type: AggregateGroupByReducers.COUNT,
+        property: null,
+        AS: 'num_total'
+      }]
+    }]
+});
+console.log(res3.results.length); // >>> 1
+console.log(res3.results); // >>>
+//[ [Object: null prototype] { type: 'bicycle', num_total: '10' } ]
+// REMOVE_START
+assert.strictEqual(res3.results.length, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START agg4
+const res4 = await client.ft.aggregate('idx:bicycle', '*', {
+  LOAD: ['__key'],
+  STEPS: [{
+      type: AggregateSteps.GROUPBY,
+      properties: '@condition',
+      REDUCE: [{
+        type: AggregateGroupByReducers.TOLIST,
+        property: '__key',
+        AS: 'bicycles'
+      }]
+    }]
+});
+console.log(res4.results.length); // >>> 3
+console.log(res4.results); // >>>
+//[[Object: null prototype] {condition: 'refurbished', bicycles: [ 'bicycle:9' ]},
+//  [Object: null prototype] {condition: 'used', bicycles: [ 'bicycle:1', 'bicycle:2', 'bicycle:3', 'bicycle:4' ]},
+//  [Object: null prototype] {condition: 'new', bicycles: [ 'bicycle:5', 'bicycle:6', 'bicycle:7', 'bicycle:0', 'bicycle:8' ]}]
+// REMOVE_START
+assert.strictEqual(res4.results.length, 3);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END

doctests/query-combined.js

@@ -0,0 +1,192 @@
+// EXAMPLE: query_combined
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient } from 'redis';
+import { SchemaFieldTypes, VectorAlgorithms } from '@redis/search';
+import { pipeline } from '@xenova/transformers';
+
+function float32Buffer(arr) {
+  const floatArray = new Float32Array(arr);
+  const float32Buffer = Buffer.from(floatArray.buffer);
+  return float32Buffer;
+}
+
+async function embedText(sentence) {
+  let modelName = 'Xenova/all-MiniLM-L6-v2';
+  let pipe = await pipeline('feature-extraction', modelName);
+
+  let vectorOutput = await pipe(sentence, {
+      pooling: 'mean',
+      normalize: true,
+  });
+
+  if (vectorOutput == null) {
+    throw new Error('vectorOutput is undefined');
+  }
+
+  const embedding = Object.values(vectorOutput.data);
+
+  return embedding;
+}
+
+let query = 'Bike for small kids';
+let vector_query = float32Buffer(await embedText('That is a very happy person'));
+
+const client = createClient();
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+    '$.description': {
+      type: SchemaFieldTypes.TEXT,
+      AS: 'description'
+    },
+    '$.condition': {
+      type: SchemaFieldTypes.TAG,
+      AS: 'condition'
+    },
+    '$.price': {
+      type: SchemaFieldTypes.NUMERIC,
+      AS: 'price'
+    },
+    '$.description_embeddings': {
+        type: SchemaFieldTypes.VECTOR,
+        TYPE: 'FLOAT32',
+        ALGORITHM: VectorAlgorithms.FLAT,
+        DIM: 384,
+        DISTANCE_METRIC: 'COSINE',
+        AS: 'vector',
+    }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+});
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_vector.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START combined1
+const res1 = await client.ft.search('idx:bicycle', '@price:[500 1000] @condition:{new}');
+console.log(res1.total); // >>> 1
+console.log(res1); // >>>
+//{
+//  total: 1,
+//  documents: [ { id: 'bicycle:5', value: [Object: null prototype] } ]
+//}
+// REMOVE_START
+assert.strictEqual(res1.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined2
+const res2 = await client.ft.search('idx:bicycle', 'kids @price:[500 1000] @condition:{used}');
+console.log(res2.total); // >>> 1
+console.log(res2); // >>>
+// {
+//   total: 1,
+//   documents: [ { id: 'bicycle:2', value: [Object: null prototype] } ]
+// }
+// REMOVE_START
+assert.strictEqual(res2.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined3
+const res3 = await client.ft.search('idx:bicycle', '(kids | small) @condition:{used}');
+console.log(res3.total); // >>> 2
+console.log(res3); // >>>
+//{
+//  total: 2,
+//  documents: [
+//    { id: 'bicycle:2', value: [Object: null prototype] },
+//    { id: 'bicycle:1', value: [Object: null prototype] }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res3.total, 2);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined4
+const res4 = await client.ft.search('idx:bicycle', '@description:(kids | small) @condition:{used}');
+console.log(res4.total); // >>> 2
+console.log(res4); // >>>
+//{
+//  total: 2,
+//  documents: [
+//    { id: 'bicycle:2', value: [Object: null prototype] },
+//    { id: 'bicycle:1', value: [Object: null prototype] }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res4.total, 2);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined5
+const res5 = await client.ft.search('idx:bicycle', '@description:(kids | small) @condition:{new | used}');
+console.log(res5.total); // >>> 3
+console.log(res5); // >>>
+//{
+//  total: 3,
+//  documents: [
+//    { id: 'bicycle:1', value: [Object: null prototype] },
+//    { id: 'bicycle:0', value: [Object: null prototype] },
+//    { id: 'bicycle:2', value: [Object: null prototype] }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res5.total, 3);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined6
+const res6 = await client.ft.search('idx:bicycle', '@price:[500 1000] -@condition:{new}');
+console.log(res6.total); // >>> 2
+console.log(res6); // >>>
+//{
+//  total: 2,
+//  documents: [
+//    { id: 'bicycle:2', value: [Object: null prototype] },
+//    { id: 'bicycle:9', value: [Object: null prototype] }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res6.total, 2);
+// REMOVE_END
+// STEP_END
+
+// STEP_START combined7
+const res7 = await client.ft.search('idx:bicycle', 
+  '(@price:[500 1000] -@condition:{new})=>[KNN 3 @vector $query_vector]', {
+    PARAMS: { query_vector: vector_query },
+    DIALECT: 2
+  }
+);
+console.log(res7.total); // >>> 2
+console.log(res7); // >>>
+//{
+//  total: 2,
+//  documents: [
+//    { id: 'bicycle:2', value: [Object: null prototype] },
+//    { id: 'bicycle:9', value: [Object: null prototype] }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res7.total, 2);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END

doctests/query-ft.js

@@ -0,0 +1,84 @@
+// EXAMPLE: query_ft
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient, SchemaFieldTypes, AggregateGroupByReducers, AggregateSteps} from 'redis';
+
+const client = createClient();
+
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+  '$.model': {
+    type: SchemaFieldTypes.TEXT,
+    AS: 'model'
+  },
+  '$.brand': {
+    type: SchemaFieldTypes.TEXT,
+    AS: 'brand'
+  },
+  '$.description': {
+    type: SchemaFieldTypes.TEXT,
+    AS: 'description'
+  }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+})
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_em.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START ft1
+const res1 = await client.ft.search('idx:bicycle', '@description: kids');
+console.log(res1.total); // >>> 2
+// REMOVE_START
+assert.strictEqual(res1.total, 2);
+// REMOVE_END
+// STEP_END
+
+// STEP_START ft2
+const res2 = await client.ft.search('idx:bicycle', '@model: ka*');
+console.log(res2.total); // >>> 1
+// REMOVE_START
+assert.strictEqual(res2.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START ft3
+const res3 = await client.ft.search('idx:bicycle', '@brand: *bikes');
+console.log(res3.total); // >>> 2
+// REMOVE_START
+assert.strictEqual(res3.total, 2);
+// REMOVE_END
+// STEP_END
+
+// STEP_START ft4
+const res4 = await client.ft.search('idx:bicycle', '%optamized%');
+console.log(res4); // >>> { total: 1, documents: [ { id: 'bicycle:3', value: [Object: null prototype] } ]}
+// REMOVE_START
+assert.strictEqual(res4.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START ft5
+const res5 = await client.ft.search('idx:bicycle', '%%optamised%%');
+console.log(res5); // >>> { total: 1, documents: [ { id: 'bicycle:3', value: [Object: null prototype] } ]}
+// REMOVE_START
+assert.strictEqual(res5.total, 1);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END

doctests/query-geo.js

@@ -0,0 +1,82 @@
+// EXAMPLE: query_geo
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient } from 'redis';
+import { SchemaFieldTypes } from '@redis/search';
+
+const client = createClient();
+
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+  '$.store_location': {
+    type: SchemaFieldTypes.GEO,
+    AS: 'store_location'
+  },
+   '$.pickup_zone': {
+    type: SchemaFieldTypes.GEOSHAPE,
+    AS: 'pickup_zone'
+  }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+})
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_em.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START geo1
+const res1= await client.ft.search('idx:bicycle', '@store_location:[-0.1778 51.5524 20 mi]');
+console.log(res1.total); // >>> 1
+console.log(res1); // >>> {total: 1, documents: [ { id: 'bicycle:5', value: [Object: null prototype] } ]}
+// REMOVE_START
+assert.strictEqual(res1.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START geo2
+const params_dict_geo2 = { bike: 'POINT(-0.1278 51.5074)' };
+const q_geo2 = '@pickup_zone:[CONTAINS $bike]';
+const res2 = await client.ft.search('idx:bicycle', q_geo2, { PARAMS: params_dict_geo2, DIALECT: 3 });
+console.log(res2.total); // >>> 1
+console.log(res2); // >>> {total: 1, documents: [ { id: 'bicycle:5', value: [Object: null prototype] } ]}
+// REMOVE_START
+assert.strictEqual(res2.total, 1);
+// REMOVE_END
+// STEP_END
+
+// STEP_START geo3
+const params_dict_geo3 = { europe: 'POLYGON((-25 35, 40 35, 40 70, -25 70, -25 35))' };
+const q_geo3 = '@pickup_zone:[WITHIN $europe]';
+const res3 = await client.ft.search('idx:bicycle', q_geo3, { PARAMS: params_dict_geo3, DIALECT: 3 });
+console.log(res3.total); // >>> 5
+console.log(res3); // >>>
+// {
+//   total: 5,
+//   documents: [
+//     { id: 'bicycle:5', value: [Object: null prototype] },
+//     { id: 'bicycle:6', value: [Object: null prototype] },
+//     { id: 'bicycle:7', value: [Object: null prototype] },
+//     { id: 'bicycle:8', value: [Object: null prototype] },
+//     { id: 'bicycle:9', value: [Object: null prototype] }
+//   ]
+// }
+// REMOVE_START
+assert.strictEqual(res3.total, 5);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END

doctests/query-range.js

@@ -0,0 +1,98 @@
+// EXAMPLE: query_range
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient, SchemaFieldTypes, AggregateGroupByReducers, AggregateSteps} from 'redis';
+
+const client = createClient();
+
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+  '$.description': {
+    type: SchemaFieldTypes.TEXT,
+    AS: 'description'
+  },
+  '$.price': {
+    type: SchemaFieldTypes.NUMERIC,
+    AS: 'price'
+  },
+  '$.condition': {
+    type: SchemaFieldTypes.TAG,
+    AS: 'condition'
+  }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+})
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_em.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START range1
+const res1 = await client.ft.search('idx:bicycle', '@price:[500 1000]');
+console.log(res1.total); // >>> 3
+// REMOVE_START
+assert.strictEqual(res1.total, 3);
+// REMOVE_END
+// STEP_END
+
+// STEP_START range2
+// FILTER is not supported
+// const res2 = await client.ft.search('idx:bicycle', '*', {
+//   FILTER: {
+//     field: 'price',
+//     min: 500,
+//     max: 1000,
+//   }
+// });
+// console.log(res2.total); // >>> 3
+// REMOVE_START
+// assert.strictEqual(res2.total, 3);
+// REMOVE_END
+// STEP_END
+
+// STEP_START range3
+// FILTER is not supported
+// const res3 = await client.ft.search('idx:bicycle', '*', {
+//   FILTER: {
+//     field: 'price',
+//     min: '(1000',
+//     max: '+inf,
+//   }
+// });
+// console.log(res3.total); // >>> 5
+// REMOVE_START
+// assert.strictEqual(res3.total, 5);
+// REMOVE_END
+// STEP_END
+
+// STEP_START range4
+const res4 = await client.ft.search(
+  'idx:bicycle',
+  '@price:[-inf 2000]',
+  {
+    SORTBY: 'price',
+    LIMIT: { from: 0, size: 5 }
+  }
+);
+console.log(res4.total); // >>> 7
+console.log(res4); // >>> { total: 7, documents: [ { id: 'bicycle:0', value: [Object: null prototype] }, { id: 'bicycle:7', value: [Object: null prototype] }, { id: 'bicycle:5', value: [Object: null prototype] }, { id: 'bicycle:2', value: [Object: null prototype] }, { id: 'bicycle:9', value: [Object: null prototype] } ] }
+// REMOVE_START
+assert.strictEqual(res4.total, 7);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END

doctests/query-vector.js

@@ -0,0 +1,111 @@
+// EXAMPLE: query_vector
+// HIDE_START
+import assert from 'node:assert';
+import fs from 'node:fs';
+import { createClient } from 'redis';
+import { SchemaFieldTypes, VectorAlgorithms } from '@redis/search';
+import { pipeline } from '@xenova/transformers';
+
+function float32Buffer(arr) {
+  const floatArray = new Float32Array(arr);
+  const float32Buffer = Buffer.from(floatArray.buffer);
+  return float32Buffer;
+}
+
+async function embedText(sentence) {
+  let modelName = 'Xenova/all-MiniLM-L6-v2';
+  let pipe = await pipeline('feature-extraction', modelName);
+
+  let vectorOutput = await pipe(sentence, {
+      pooling: 'mean',
+      normalize: true,
+  });
+
+  const embedding = Object.values(vectorOutput?.data);
+
+  return embedding;
+}
+
+let query = 'Bike for small kids';
+let vector_query = float32Buffer(await embedText('That is a very happy person'));
+
+const client = createClient();
+await client.connect().catch(console.error);
+
+// create index
+await client.ft.create('idx:bicycle', {
+    '$.description': {
+      type: SchemaFieldTypes.TEXT,
+      AS: 'description'
+    },
+    '$.description_embeddings': {
+        type: SchemaFieldTypes.VECTOR,
+        TYPE: 'FLOAT32',
+        ALGORITHM: VectorAlgorithms.FLAT,
+        DIM: 384,
+        DISTANCE_METRIC: 'COSINE',
+        AS: 'vector'
+    }
+}, {
+    ON: 'JSON',
+    PREFIX: 'bicycle:'
+});
+
+// load data
+const bicycles = JSON.parse(fs.readFileSync('data/query_vector.json', 'utf8'));
+
+await Promise.all(
+  bicycles.map((bicycle, bid) => {
+      return client.json.set(`bicycle:${bid}`, '$', bicycle);
+  })
+);
+// HIDE_END
+
+// STEP_START vector1
+const res1 = await client.ft.search('idx:bicycle', 
+  '*=>[KNN 3 @vector $query_vector AS score]', {
+    PARAMS: { query_vector: vector_query },
+    RETURN: ['description'],
+    DIALECT: 2
+  }
+);
+console.log(res1.total); // >>> 3
+console.log(res1); // >>>
+//{
+//  total: 3,
+//  documents: [
+//    { id: 'bicycle:0', value: [Object: null prototype] {} },
+//    { id: 'bicycle:2', value: [Object: null prototype] {} },
+//    { id: 'bicycle:9', value: [Object: null prototype] {} }
+//  ]
+//}
+// REMOVE_START
+assert.strictEqual(res1.total, 3);
+// REMOVE_END
+// STEP_END
+
+// STEP_START vector2
+const res2 = await client.ft.search('idx:bicycle', 
+  '@vector:[VECTOR_RANGE 0.9 $query_vector]=>{$YIELD_DISTANCE_AS: vector_dist}', {
+    PARAMS: { query_vector: vector_query },
+    SORTBY: 'vector_dist',
+    RETURN: ['vector_dist', 'description'],
+    DIALECT: 2
+  }
+);
+console.log(res2.total); // >>> 1
+console.log(res2); // >>>
+//{
+//  total: 1,
+//  documents: [ { id: 'bicycle:0', value: [Object: null prototype] } ]
+//}
+// REMOVE_START
+assert.strictEqual(res2.total, 1);
+// REMOVE_END
+// STEP_END
+
+// REMOVE_START
+// destroy index and data
+await client.ft.dropIndex('idx:bicycle', { DD: true });
+await client.disconnect();
+// REMOVE_END