database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-23 21:49:02 +03:00
Code Activity
Files
curly-brace-expr
sqlite/ext/vec1/vec1.c
dan a27bcfe21c Fix a vec1 problem where rounding errors when assigning vectors to centroids could cause delete operations to fail. 
FossilOrigin-Name: aab150f4b39aa8dfa0df9161368b5f3b50859b2657be1bf55d06b67a03005c6b
2026-10-10 18:06:06 +00:00

13111 lines
369 KiB
C
Raw Permalink Blame History

/*
** 2026-02-04
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file contains the SQLite vector ANN extension "vec1". To build
** as a loadable module on x86_64 with gcc:
**
** gcc -O3 -DNDEBUG -mavx2 -mfma vec1.c -shared -fPIC -o vec1.so
**
** Documentation available at:
**
** https://sqlite.org/vec1
*/
/*
** The two distance metrics supported by the system.
*/
#define VEC1_DISTANCE_L2 1
#define VEC1_DISTANCE_COS 2
/*
** Default values for training parameters.
*/
#define VEC1_TRAINING_DEFAULT_CODESIZE 0
#define VEC1_TRAINING_DEFAULT_NBUCKET 0
#define VEC1_TRAINING_DEFAULT_SVD_VERIFY 0
#define VEC1_TRAINING_DEFAULT_OPQ 0
#define VEC1_TRAINING_DEFAULT_NOPQ_ROUND 5
#define VEC1_TRAINING_DEFAULT_RESIDUAL 1
#define VEC1_TRAINING_DEFAULT_DISTANCE VEC1_DISTANCE_L2
/*
** Max and min values for various parameters.
**
** VEC1_MAX_K:
** At present, a KANN query with an index allocates K*16 bytes for
** the heap data structure used to accumulate the K best matches.
** In its default configuration, sqlite3_malloc() is limited to
** 1GB allocations. So limit K so as to at least avoid SQLITE_NOMEM
** errors in the default configuration.
*/
#define VEC1_MAX_K 32*1024*1024
/*
** Software version. Returned by "SELECT vec1_info()".
*/
#define VEC1_VERSION "0.8"
/*
** Values for Vec1ModelHeader.iVersion:
**
** VEC1_HEADER_VERSION:
** Used by default, if nothing below applies.
**
** VEC1_HEADER_VERSION_BIN:
** Used if the model uses a binary-quantizer.
*/
#define VEC1_HEADER_VERSION (0*1000 + 4)
#define VEC1_HEADER_VERSION_BIN (0*1000 + 7)
/*
** Shadow tables version. Stored in %_config table. Corresponds to the
** minimum version that matches the current shadow table setup.
*/
#define VEC1_CURRENT_FMTVERSION (0*1000 + 4)
#include "sqlite3.h"
/*
** Unless VEC1_STATIC is defined, compile as a loadable extension.
*/
#ifndef VEC1_STATIC
# include "sqlite3ext.h"
SQLITE_EXTENSION_INIT3
#endif
/* System include files */
# include <assert.h>
# include <string.h>
# include <stdarg.h>
# include <math.h>
# include <stdio.h>
# include <stdlib.h>
/* Standard sized types. */
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned short u16;
typedef float f32;
typedef int i32;
typedef char i8;
#if !defined(VEC1_TEST_SCALAR)
#define VERTICAL_SUM_GENERIC(aAcc, nAcc, I) { \
int dd; \
for(dd=4; dd<nAcc; dd++) aAcc[dd & 3] = I (aAcc[dd & 3], aAcc[dd]); \
for(dd=2; dd<4 && dd<nAcc; dd++) aAcc[dd & 1] = I (aAcc[dd & 1], aAcc[dd]); \
if( nAcc>1 ) aAcc[0] = I (aAcc[0], aAcc[1]); \
}
#ifdef __AVX2__
# define VEC1_HAVE_AVX2 1
# define VEC1_TRANSPOSE_WIDTH 32
# define VEC1_NTRANSPOSE_WIDTH 8
/*
** Dimensions used for vec1BestMatchN() and vec1BestMatchTransposeN().
*/
# define VEC1_MULTIMATCH_NVEC 8
# define VEC1_MULTIMATCH_NCENT 8
/* AVX2 builds should use transposed arrays of centroids where possible.
** This avoids horizontal sums (expensive by design on AVX2) and frees
** up registers, allowing more concurrent FMA and other operations. */
# define VEC1_USE_TRANSPOSITION 1
/*
** When assembling the covariance matrix (part of OPQ rotation training),
** deal with (VEC1_COVAR_MULTIMATCH_MULT * VEC1_MULTIMATCH_NVEC) vectors
** at a time. See vec1CovarianceWork() for details.
*/
# define VEC1_COVAR_MULTIMATCH_MULT 1
# include <immintrin.h>
# if defined(__FMA__) || defined(_MSC_VER)
/* With MSVC, __AVX2__ implies __FMA__ */
# define FMADD(a,b,c) _mm256_fmadd_ps((a),(b),(c))
# define FMADD64(a,b,c) _mm256_fmadd_pd((a),(b),(c))
# else
# define FMADD(a,b,c) _mm256_add_ps((c), _mm256_mul_ps((a),(b)))
# define FMADD64(a,b,c) _mm256_add_pd((c), _mm256_mul_pd((a),(b)))
# endif
/* Load 8 unaligned floats. Or 4 unaligned doubles. */
# define LOADU(a) _mm256_loadu_ps(a)
# define LOADU64(a) _mm256_loadu_pd(a)
/*
** Parameter vec is a __m256 register containing 8 floats. Parameter
** s is a float. This macro sets s to the sum of the 8 values in vec. i.e.
**
** s = vec[0] + vec[1] + vec[2] + vec[3]
** + vec[4] + vec[5] + vec[6] + vec[7];
*/
# define HORIZONTAL_SUM(s, vec) \
{ \
__m128 vc_high = _mm256_extractf128_ps(vec, 1); \
__m128 vc_low = _mm256_castps256_ps128(vec); \
__m128 vc128 = _mm_add_ps(vc_high, vc_low); \
vc128 = _mm_hadd_ps(vc128, vc128); \
vc128 = _mm_hadd_ps(vc128, vc128); \
s = _mm_cvtss_f32(vc128); \
}
/*
** Parameter vec is a __m256 register containing 4 doubles. Parameter
** s is a double. This macro sets s to the sum of the 4 values in vec. i.e.
**
** s = vec[0] + vec[1] + vec[2] + vec[3]
*/
# define HORIZONTAL_SUM64(s, vec) \
{ \
__m128d _lo = _mm256_castpd256_pd128(vec); \
__m128d _hi = _mm256_extractf128_pd(vec, 1); \
__m128d _sum = _mm_add_pd(_lo, _hi); \
(s) = _mm_cvtsd_f64(_mm_hadd_pd(_sum, _sum)); \
}
/*
** aAcc[] is an nAcc element array of type _m256 containing 8 32-bit floats.
** This macro sums the vectors and writes the results to aAcc[0].
*/
# define VERTICAL_SUM(aAcc, nAcc) VERTICAL_SUM_GENERIC(aAcc,nAcc,_mm256_add_ps)
#elif defined(__aarch64__)
# define VEC1_HAVE_NEON 1
# define VEC1_TRANSPOSE_WIDTH 32
# define VEC1_MULTIMATCH_NVEC 4
# define VEC1_MULTIMATCH_NCENT 4
# define VEC1_COVAR_MULTIMATCH_MULT 4
# include <arm_neon.h>
/*
** aAcc[] is an nAcc element array of type float32x4. This macro sums the
** vectors and writes the results to aAcc[0].
*/
#define VERTICAL_SUM(aAcc, nAcc) VERTICAL_SUM_GENERIC(aAcc, nAcc, vaddq_f32)
#endif
#endif /* VEC1_TEST_SCALAR */
#ifndef VEC1_TRANSPOSE_WIDTH
/* Scalar fallback. Even scalar needs transpose widths. */
# define VEC1_TRANSPOSE_WIDTH 8
# define VEC1_MULTIMATCH_NVEC 4
# define VEC1_MULTIMATCH_NCENT 4
# define VEC1_COVAR_MULTIMATCH_MULT 1
#endif
#define VEC1_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
#define VEC1_SMALLEST_INT64 (((i64)-1) - VEC1_LARGEST_INT64)
#define SWAP(T, a, b) { T tmp; tmp = a; a = b; b = tmp; }
/*
** Return a hardware counter value. These are used to calculate the
** percentages in the performance reports from vec1cat.qprofile and the
** final progress callback from within vec1_train().
*/
static u64 vec1HardwareTimer(void){
u64 ret = 0;
#if !defined(__STRICT_ANSI__)
# if defined(__aarch64__)
__asm__ __volatile__("mrs %0, cntvct_el0" : "=r"(ret));
# elif defined(__GNUC__) && defined(__x86_64__)
unsigned int lo, hi;
__asm__ __volatile__("rdtscp" : "=a"(lo), "=d"(hi) :: "%rcx");
ret = ((u64)hi << 32) | lo;
# endif
#endif
return ret;
}
/*
** Figure out which threading model to use. Define VEC1_THREADS_WINDOWS
** to use windows threads, VEC1_THREADS_PTHREADS to use pthreads, or
** neither to use no threads.
*/
#if !defined(VEC1_THREADS) || VEC1_THREADS!=0
# if defined(_WIN32)
# define VEC1_THREADS_WINDOWS 1
# define VEC1_THREADS 1
# elif defined(__unix__) || defined(__APPLE__) || defined(__linux__)
# define VEC1_THREADS_PTHREADS
# define VEC1_THREADS 1
# else
# define VEC1_THREADS 0
# endif
#endif
#define size_of_array(X) ((int)(sizeof(X)/sizeof((X)[0])))
#define MAX(a,b) ((a<b) ? (b) : (a))
#define MIN(a,b) ((a>b) ? (b) : (a))
#define UNUSED_PARAMETER(x) (void)(x)
#define UNUSED_PARAMETER2(x,y) (void)(x),(void)(y)
typedef struct Vec1Tab Vec1Tab;
typedef struct Vec1TabList Vec1TabList;
/*
** An instance of the following object is allocated when registering
** the vec1 module with a database handle. A pointer is passed as the
** the context pointer to sqlite3_create_module_v2(). It maintains a
** list of all Vec1Tab tables attached to the database handle.
**
** This object does not need a mutex, as it belongs to a single db handle.
** The db handle mutex ensures exclusive access.
*/
struct Vec1TabList {
int nRef;
sqlite3 *db;
Vec1Tab *pFirst;
int nThread; /* Number of threads to use for various ops */
double nProbeArg; /* Default "nprobe" for queries */
};
/*
** This structure represents a model-header. When serialized, the format
** is a packed array of six 32-bit big-endian integers, in the order
** shown in the struct below.
*/
typedef struct Vec1ModelHeader Vec1ModelHeader;
struct Vec1ModelHeader {
u32 iVersion; /* Version number currently 0004 */
u32 flags;
u32 nElem; /* Size of vectors in elements */
u32 nCodebook; /* Number of codebooks for PQ */
u32 nBucket; /* Number of buckets for IVF */
u32 eDistance; /* A VEC1_DISTANCE_XXX value */
};
#define VEC1_HEADER_SIZE (6*sizeof_u32)
/*
** Candidate values for the "flags" field of a Vec1ModelHeader. As follows:
**
** VEC1_MODEL_INDEX:
** Set for any configuration except "none".
**
** VEC1_MODEL_ROTATE:
** Set if model includes a full rotation matrix.
**
** VEC1_MODEL_RESIDUAL:
** Set if residual, not full, vectors are encoded.
**
** VEC1_MODEL_BITQUANT:
** Set to use bitwise quantization instead of PQ. Compressed vectors are
** a 32-bit float (the norm) followed by bitwise sketch of the vector
** or residual vector.
**
** VEC1_MODEL_WHT:
** Do Walsh Hadamard transform.
*/
#define VEC1_MODEL_INDEX 0x01
#define VEC1_MODEL_ROTATE 0x02
#define VEC1_MODEL_RESIDUAL 0x04
#define VEC1_MODEL_BITQUANT 0x08
#define VEC1_MODEL_WHT 0x10
/*
** Use these instead of actual sizeof() macros to avoid signed/unsigned
** compiler warnings.
*/
#define sizeof_u16 2
#define sizeof_u32 4
#define sizeof_f32 4
#define sizeof_f64 8
/*
** The three types of vector elements (that will eventually be) supported.
*/
#define VEC1_TYPE_FLOAT32 0
#define VEC1_TYPE_INT32 1
#define VEC1_TYPE_INT8 2
/*
** Maximum allowed threads for both vec1 and vec1_train().
*/
#define VEC1_MAX_NTHREAD 128
static int vec1CorruptError(){
return SQLITE_CORRUPT_VTAB;
}
#define VEC1_CORRUPT vec1CorruptError();
static void vec1PutU32(u8 *aBuf, u32 val){
aBuf[0] = (val >> 24) & 0xFF;
aBuf[1] = (val >> 16) & 0xFF;
aBuf[2] = (val >> 8) & 0xFF;
aBuf[3] = (val >> 0) & 0xFF;
}
/*
** Write a model header into buffer aBuf[].
*/
static void vec1HeaderWrite(
u8 *aBuf,
int flags,
int nElem,
int nCodebook,
int nBucket,
int eDistance
){
/* VEC1_HEADER_VERSION_BIN if the model uses a binary quantizer,
** VEC1_HEADER_VERSION otherwise. */
u32 eHdrVersion = VEC1_HEADER_VERSION;
if( flags & VEC1_MODEL_BITQUANT ){
eHdrVersion = VEC1_HEADER_VERSION_BIN;
}
vec1PutU32(&aBuf[0], eHdrVersion);
vec1PutU32(&aBuf[4], flags);
vec1PutU32(&aBuf[8], nElem);
vec1PutU32(&aBuf[12], nCodebook);
vec1PutU32(&aBuf[16], nBucket);
vec1PutU32(&aBuf[20], eDistance);
}
static u32 vec1GetU32(const u8 *aBuf){
return (((u32)aBuf[0]) << 24)
+ (((u32)aBuf[1]) << 16)
+ (((u32)aBuf[2]) << 8)
+ (((u32)aBuf[3]) << 0);
}
/*
** Deserialize buffer aBuf into object p.
*/
static void vec1HeaderRead(const u8 *aBuf, Vec1ModelHeader *p){
p->iVersion = vec1GetU32(&aBuf[0]);
p->flags = vec1GetU32(&aBuf[4]);
p->nElem = vec1GetU32(&aBuf[8]);
p->nCodebook = vec1GetU32(&aBuf[12]);
p->nBucket = vec1GetU32(&aBuf[16]);
p->eDistance = vec1GetU32(&aBuf[20]);
}
/*
** Version of sqlite3_free() that can be used as a destructor function with
** things like sqlite3_bind_blob().
*/
static void vec1SqliteFree(void *x){
sqlite3_free(x);
}
typedef struct Vec1AnnResult Vec1AnnResult;
struct Vec1AnnResult {
sqlite3_int64 iRowid;
double fDist;
};
typedef struct Vec1AnnHeap Vec1AnnHeap;
struct Vec1AnnHeap {
Vec1AnnResult *aRes; /* Heap array */
i64 nRes; /* Current number of elements */
i64 nMax; /* Capacity */
i64 nAlloc; /* Current allocated size of aRes[] */
int rc; /* Error code (perhaps SQLITE_NOMEM) */
int bStreaming;
double fMin; /* Current threshold for admittance */
};
typedef struct Vec1Buffer Vec1Buffer;
struct Vec1Buffer {
unsigned char *a;
int n;
int nAlloc;
};
/*
** Return the dot-product of vectors A and B, each d elements in size.
*/
static float vec1DotProduct(
int d,
const float *A,
const float *B
){
float sum = 0.0f;
int j = 0;
#ifdef VEC1_HAVE_AVX2
# ifndef VEC1_DOT_NACC
# define VEC1_DOT_NACC 4
# endif
const int nAcc = VEC1_DOT_NACC;
int cc;
/* Create and initialize to zero nAcc accumulator registers. */
__m256 acc[VEC1_DOT_NACC];
for(cc=0; cc<nAcc; cc++) acc[cc] = _mm256_setzero_ps();
for(; j<=(d-(8*nAcc)); j+=(8*nAcc)){
for(cc=0; cc<nAcc; cc++){
acc[cc] = FMADD(LOADU(&A[j+cc*8]), LOADU(&B[j+cc*8]), acc[cc]);
}
}
/* Sum all nAcc accumulators into acc[0]. */
VERTICAL_SUM(acc, nAcc);
/* Process the part of the tail that can be done 8 elements at a time
** using acc[0] as the accumulator. Then sum all 8 elements of acc[0]
** into scalar float "sum". */
for(; j<=(d-8); j+=8){
acc[0] = FMADD(LOADU(&A[j]), LOADU(&B[j]), acc[0]);
}
HORIZONTAL_SUM(sum, acc[0]);
#endif
#ifdef VEC1_HAVE_NEON
# ifndef VEC1_DOT_NACC
# define VEC1_DOT_NACC 8
# endif
const int nAcc = VEC1_DOT_NACC;
int cc;
/* Create and initialize to zero nAcc accumulator registers. */
float32x4_t acc[VEC1_DOT_NACC];
for(cc=0; cc<nAcc; cc++) acc[cc] = vdupq_n_f32(0.0f);
for(; j<=(d-(4*nAcc)); j+=(4*nAcc)){
for(cc=0; cc<nAcc; cc++){
float32x4_t a = vld1q_f32(&A[j+cc*4]);
float32x4_t b = vld1q_f32(&B[j+cc*4]);
acc[cc] = vfmaq_f32(acc[cc], a, b);
}
}
/* Sum all nAcc accumulators into acc[0]. */
VERTICAL_SUM(acc, nAcc);
/* Process the part of the tail that can be done 4 elements at a time
** using acc[0] as the accumulator. Then sum all 4 elements of acc[0]
** into scalar float "sum". */
for(; j<=(d-4); j+=4){
acc[0] = vfmaq_f32(acc[0], vld1q_f32(&A[j]), vld1q_f32(&B[j]));
}
sum = vaddvq_f32(acc[0]);
#endif
for(; j<d; j++){
sum += A[j] * B[j];
}
return sum;
}
/*
** Rotate x -> y using row-major matrix A.
*/
static void vec1RotateVector(
int d,
const float* A, /* d x d, row-major */
const float* x, /* input vector, length d */
float* y /* output vector, length d */
){
int i;
for(i=0; i<d; i++){
y[i] = vec1DotProduct(d, &A[i*d], x);
}
}
/*
** Normalize nElem dimension vector aElem.
**
** Assume a vector with magnitude 0.0 normalizes to itself.
*/
static void vec1NormalizeVector(float *aElem, int nElem){
int ii;
double fSum = vec1DotProduct(nElem, aElem, aElem);
if( fSum==0.0 ) return;
fSum = 1.0 / sqrt(fSum);
for(ii=0; ii<nElem; ii++){
aElem[ii] = (float)(aElem[ii] * fSum);
}
}
#define VEC1_INITIAL_STREAMING_ALLOC ((128*1024)/16)
/*
** Initialize a new, existing, heap object.
*/
static int vec1HeapInit(Vec1AnnHeap *p, i64 nMax, int bStreaming){
i64 nAlloc = (bStreaming ? VEC1_INITIAL_STREAMING_ALLOC : nMax);
nAlloc = MAX(nAlloc, nMax);
p->aRes = (Vec1AnnResult *)sqlite3_malloc64(sizeof(Vec1AnnResult) * nAlloc);
if( p->aRes==0 ){
return SQLITE_NOMEM;
}
p->nRes = 0;
p->nMax = nMax;
p->nAlloc = nAlloc;
p->bStreaming = bStreaming;
p->fMin = INFINITY;
return SQLITE_OK;
}
/*
** A new element, element i, has just been added to the heap as a new leaf
** node. This function repairs the heap by bubbling-up from the new node
** to the root.
*/
static void vec1HeapBubbleUp(Vec1AnnHeap *p, int i){
int idx = i;
while( idx>0 ){
int idxParent = (idx-1)/2;
if( p->aRes[idxParent].fDist>p->aRes[idx].fDist ) break;
SWAP(Vec1AnnResult, p->aRes[idx], p->aRes[idxParent]);
idx = idxParent;
}
}
/*
** A new element has just replaced the heap root node (i.e. the previous
** entry with the largest distance in the heap has been replaced by a
** smaller distance. This function repairs the heap by bubbling-down from
** the new root towards the leaves.
*/
static void vec1HeapBubbleDown(Vec1AnnHeap *p){
int idx = 0;
for(;;){
int idxLeft = idx*2 + 1;
int idxRight = idxLeft + 1;
int idxMax = idx;
if( idxLeft<p->nMax && p->aRes[idxLeft].fDist>p->aRes[idxMax].fDist ){
idxMax = idxLeft;
}
if( idxRight<p->nMax && p->aRes[idxRight].fDist>p->aRes[idxMax].fDist ){
idxMax = idxRight;
}
if( idxMax==idx ) break;
SWAP(Vec1AnnResult, p->aRes[idx], p->aRes[idxMax]);
idx = idxMax;
}
}
/*
** Add an entry to the heap structure. If the heap is already full, the
** entry with the largest fDist value seen so far is discarded.
*/
static void vec1HeapInsert(Vec1AnnHeap *p, sqlite3_int64 iRowid, double fDist){
if( p->nRes<p->nMax ){
p->aRes[p->nRes].iRowid = iRowid;
p->aRes[p->nRes].fDist = fDist;
vec1HeapBubbleUp(p, (int)p->nRes);
p->nRes++;
}else if( p->bStreaming ){
if( p->nRes>=p->nAlloc ){
i64 nNew = p->nAlloc * 2;
Vec1AnnResult *aNew = 0;
if( p->rc==SQLITE_OK ){
i64 nByte = nNew*sizeof(Vec1AnnResult);
aNew = (Vec1AnnResult*)sqlite3_realloc64(p->aRes, nByte);
}
if( aNew==0 ){
p->rc = SQLITE_NOMEM;
return;
}
p->aRes = aNew;
p->nAlloc = nNew;
}
p->aRes[p->nRes].iRowid = iRowid;
p->aRes[p->nRes].fDist = fDist;
if( fDist<p->aRes[0].fDist ){
SWAP(Vec1AnnResult, p->aRes[0], p->aRes[p->nRes]);
vec1HeapBubbleDown(p);
}
p->nRes++;
}else if( fDist<p->aRes[0].fDist ){
p->aRes[0].iRowid = iRowid;
p->aRes[0].fDist = fDist;
vec1HeapBubbleDown(p);
p->fMin = p->aRes[0].fDist;
}
}
/*
** Ensure there is room in the buffer for another nByte bytes of data. That
** is, make sure the allocated size is at least (pBuf->n + nByte).
**
** Return SQLITE_OK if successful, or SQLITE_NOMEM if an OOM error is
** encountered.
*/
static int vec1BufferGrow(Vec1Buffer *pBuf, sqlite3_int64 nByte){
if( nByte>pBuf->nAlloc-pBuf->n ){
sqlite3_int64 nNew = pBuf->nAlloc;
unsigned char *aNew = 0;
if( nNew==0 ) nNew = nByte + 100;
while( nNew < (pBuf->n + nByte) ){
nNew = nNew * 2;
}
aNew = sqlite3_realloc64(pBuf->a, nNew);
if( aNew==0 ){
return SQLITE_NOMEM;
}
pBuf->a = aNew;
pBuf->nAlloc = (int)nNew;
}
return SQLITE_OK;
}
/*
** Ensure that the buffer passed as the first argument has at least nByte
** bytes of space allocated.
*/
static int vec1BufferSize(Vec1Buffer *pBuf, sqlite3_int64 nByte){
if( nByte>pBuf->nAlloc ){
u8 *aNew = sqlite3_realloc64(pBuf->a, nByte);
if( aNew==0 ){
return SQLITE_NOMEM;
}
pBuf->a = aNew;
pBuf->nAlloc = (int)nByte;
}
return SQLITE_OK;
}
/*
** Free the allocation and zero the buffer structure.
*/
static void vec1BufferFree(Vec1Buffer *pBuf){
sqlite3_free(pBuf->a);
memset(pBuf, 0, sizeof(Vec1Buffer));
}
/*
** Set the error message in context object pCtx to the result of formatting
** zFmt printf() style.
*/
static void vec1ResultErrorF(
sqlite3_context *pCtx, /* Context to set error in */
const char *zFmt, /* printf() style format for error message */
... /* Trailing arguments for printf() */
){
char *zMsg = 0;
va_list ap;
va_start(ap, zFmt);
zMsg = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
sqlite3_result_error(pCtx, zMsg, -1);
sqlite3_free(zMsg);
}
/*
** Attempt to allocate and zero a block of nByte bytes of memory. If
** succesful, return a pointer to the new allocation. It is the responsibility
** of the caller to eventually free the allocation using sqlite3_free().
**
** If the allocation attempt fails, return 0 and leave an OOM error in
** context object pCtx.
*/
static void *vec1ContextMalloc(sqlite3_context *pCtx, sqlite3_int64 nByte){
void *pRet = sqlite3_malloc64(nByte);
if( pRet==0 ){
sqlite3_result_error_nomem(pCtx);
}else{
memset(pRet, 0, nByte);
}
return pRet;
}
/*
** Implementation of scalar function vec1_info().
*/
static void vec1InfoFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
char *z = 0;
const char *zSimd = 0;
const char *zThreads = 0;
UNUSED_PARAMETER2(nVal, aVal);
#if defined(VEC1_HAVE_AVX2)
zSimd = "AVX2";
#elif defined(VEC1_HAVE_NEON)
zSimd = "NEON";
#else
zSimd = "Scalar";
#endif
#if VEC1_THREADS
zThreads = "multi-threaded";
#else
zThreads = "single-threaded";
#endif
z = sqlite3_mprintf("version %s (%s, %s)", VEC1_VERSION, zSimd, zThreads);
if( z==0 ){
sqlite3_result_error_nomem(pCtx);
}else{
sqlite3_result_text(pCtx, z, -1, vec1SqliteFree);
}
}
/*
** Return true if character c counts as whitespace. This is used
** by vec1_from_json().
*/
static int vec1_isspace(char c){
return (c==' ');
}
/*
** Implementation of scalar function vec1_from_json(). Converts a json array
** of numbers into a vector in vec1 native format (32-bit little-endian
** floats).
*/
static void vec1FromJsonFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
const char *zJson = (const char*)sqlite3_value_text(aVal[0]);
int nJson = sqlite3_value_bytes(aVal[0]);
const char *p = zJson;
assert( nVal==1 );
UNUSED_PARAMETER(nVal);
float *aVec = 0;
int nVec = 0;
/* vec1_from_json(NULL) returns NULL */
if( sqlite3_value_type(aVal[0])==SQLITE_NULL ) return;
/* Allocate space for the return value */
aVec = (float*)sqlite3_malloc(sizeof_f32*(nJson/2));
if( !aVec ){
sqlite3_result_error_nomem(pCtx);
return;
}
while( vec1_isspace(*p) ) p++;
if( *p!='[' ) goto parse_failed;
p++;
while( 1 ){
char *p2 = 0;
double rVal = 0.0;
while( vec1_isspace(*p) ) p++;
rVal = strtod(p, &p2);
if( p==p2 ) goto parse_failed;
p = p2;
aVec[nVec] = (float)rVal;
nVec++;
while( vec1_isspace(*p) ) p++;
if( *p==']' ){
p++;
break;
}
if( *p!=',' ) goto parse_failed;
p++;
}
while( vec1_isspace(*p) ) p++;
if( *p ) goto parse_failed;
sqlite3_result_blob(pCtx, (void*)aVec, nVec*sizeof_f32, vec1SqliteFree);
return;
parse_failed:
sqlite3_free(aVec);
sqlite3_result_error(pCtx, "error parsing json vector", -1);
}
/*
** SQL Scalar Functions:
**
** vec1_to_json(BLOB)
** vec1_to_json_f(BLOB)
** vec1_to_json_i(BLOB)
*/
static void vec1ToJsonGeneric(
sqlite3_context *pCtx,
int eType,
int nBytePerElem,
sqlite3_value *pVal
){
int ii;
sqlite3 *db = sqlite3_context_db_handle(pCtx);
sqlite3_str *pStr = 0;
int n = sqlite3_value_bytes(pVal);
const void *a = sqlite3_value_blob(pVal);
if( n==0 || (n%nBytePerElem)!=0 ){
sqlite3_result_error(pCtx, "vec1: invalid vector argument", -1);
return;
}
n = n / nBytePerElem;
pStr = sqlite3_str_new(db);
if( eType==VEC1_TYPE_FLOAT32 ){
const float *aFloat = (const float*)a;
sqlite3_str_appendf(pStr, "[%.9g", aFloat[0]);
for(ii=1; ii<n; ii++){
sqlite3_str_appendf(pStr, ",%.9g", aFloat[ii]);
}
#if 0
}else if( eType==VEC1_TYPE_INT8 ){
const i8 *aInt = (const i8*)a;
sqlite3_str_appendf(pStr, "[%d", (int)aInt[0]);
for(ii=1; ii<n; ii++){
sqlite3_str_appendf(pStr, ",%d", (int)aInt[ii]);
}
#endif
}else{
const int *aInt = (const int*)a;
assert( eType==VEC1_TYPE_INT32 );
sqlite3_str_appendf(pStr, "[%d", aInt[0]);
for(ii=1; ii<n; ii++){
sqlite3_str_appendf(pStr, ",%d", aInt[ii]);
}
}
sqlite3_str_appendall(pStr, "]");
if( sqlite3_str_errcode(pStr)!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, sqlite3_str_errcode(pStr));
}else{
sqlite3_result_text(pCtx, sqlite3_str_value(pStr), -1, vec1SqliteFree);
}
sqlite3_str_finish(pStr);
}
/*
** Scalar function vec1_to_json_f().
*/
static void vec1ToJsonFFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
assert( nVal==1 );
UNUSED_PARAMETER(nVal);
vec1ToJsonGeneric(pCtx, VEC1_TYPE_FLOAT32, sizeof_f32, aVal[0]);
}
/*
** Scalar function vec1_to_json_i().
*/
static void vec1ToJsonIFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
assert( nVal==1 );
UNUSED_PARAMETER(nVal);
vec1ToJsonGeneric(pCtx, VEC1_TYPE_INT32, sizeof(int), aVal[0]);
}
/*
** This function is used to parse the small json objects used to specify
** configuration options to components in this module.
**
** For each key-value pair in json object zJson, the callback x is
** invoked once, as follows:
**
** x(pCtx, zKey, eType, iValue, fValue, zValue, pzErr)
**
** Parameters are as follows:
**
** pCtx: Copy of pCtx passed to this function.
** zKey: Name of key.
** eType: One of SQLITE_INTEGER, SQLITE_FLOAT or SQLITE_TEXT.
** iValue: For SQLITE_INTEGER, the value. Otherwise undefined.
** fValue: For SQLITE_INTEGER or SQLITE_FLOAT, the value. Or undefined.
** zValue: For SQLITE_TEXT the value. Or undefined.
** pzErr: Output variable used by callback to set an error message.
**
** If x() returns other than SQLITE_OK, processing stops and the error
** code is returned to the caller. In this case x() may set *pzErr to
** an sqlite3_malloc() buffer containing an error message.
*/
static int vec1ParseJsonConfig(
sqlite3 *db,
const char *zJson,
int (*x)(
void *pCtx,
const char *zKey,
int eType,
i64 iValue,
double fValue,
const char *zValue,
char **pzErr
),
void *pCtx,
char **pz
){
sqlite3_stmt *pStmt = 0;
int rc = SQLITE_OK;
int rc2 = SQLITE_OK;
rc = sqlite3_prepare_v2(
db, "SELECT key, value FROM json_each(?)", -1, &pStmt, 0
);
if( rc==SQLITE_OK ){
rc = sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
double fVal = 0;
i64 iVal = 0;
int eType = sqlite3_value_numeric_type(sqlite3_column_value(pStmt, 1));
switch( eType ){
case SQLITE_INTEGER:
iVal = sqlite3_column_int64(pStmt, 1);
fVal = (double)iVal;
break;
case SQLITE_FLOAT:
fVal = sqlite3_column_double(pStmt, 1);
if( fVal==(double)(i64)fVal ){
iVal = (i64)fVal;
eType = SQLITE_INTEGER;
}
break;
case SQLITE_TEXT:
break;
default:
/* We don't want any NULL values. Or blob, if that is even possible. */
*pz = sqlite3_mprintf("vec1: null value in json object");
rc = SQLITE_ERROR;
break;
}
if( rc==SQLITE_OK ){
const char *zVal = (const char*)sqlite3_column_text(pStmt, 1);
rc = x(pCtx, zKey, eType, iVal, fVal, zVal, pz);
}
}
rc2 = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK && rc2!=SQLITE_OK ){
rc = rc2;
*pz = sqlite3_mprintf("%s", sqlite3_errmsg(db));
}
return rc;
}
/*
** Write val into aBuf[] as a 64-bit unsigned big-endian integer.
*/
static void vec1PutU64(u8 *aBuf, u64 val){
aBuf[0] = (val >> 56) & 0xFF;
aBuf[1] = (val >> 48) & 0xFF;
aBuf[2] = (val >> 40) & 0xFF;
aBuf[3] = (val >> 32) & 0xFF;
aBuf[4] = (val >> 24) & 0xFF;
aBuf[5] = (val >> 16) & 0xFF;
aBuf[6] = (val >> 8) & 0xFF;
aBuf[7] = (val >> 0) & 0xFF;
}
/*
** Read and return a 64-bit unsigned big-endian integer from buffer aBuf[].
*/
static u64 vec1GetU64(const u8 *aBuf){
return (((u64)aBuf[0]) << 56)
+ (((u64)aBuf[1]) << 48)
+ (((u64)aBuf[2]) << 40)
+ (((u64)aBuf[3]) << 32)
+ (((u64)aBuf[4]) << 24)
+ (((u64)aBuf[5]) << 16)
+ (((u64)aBuf[6]) << 8)
+ (((u64)aBuf[7]) << 0);
}
/*
** Subtract a2 from a1 and leave the result in aOut. i.e.
**
** aOut = a1 - a2
*/
static void vec1Sub(float *aOut, const float *a1, const float *a2, int n){
int i;
for(i=0; i<n; i++){
aOut[i] = a1[i] - a2[i];
}
}
/*
** Subtract vector a2 from a1. i.e. do:
**
** a1 = a1 - a2
*/
static void vec1SubInPlace(float *a1, const float *a2, int n){
vec1Sub(a1, a1, a2, n);
}
/*
** Return the L2 distance between vectors a1[] and a2[], each n elements
** in size.
*/
static double vec1L2Dist(
const float *a1,
const float *a2,
int n
){
double ret = 0.0f;
int i = 0;
#ifdef VEC1_HAVE_AVX2
# ifndef VEC1_L2_NACC
# define VEC1_L2_NACC 4
# endif
const int nAcc = VEC1_L2_NACC;
int c;
/* Create and initialize to zero nAcc accumulator registers. */
__m256 acc[VEC1_L2_NACC];
for(c=0; c<nAcc; c++) acc[c] = _mm256_setzero_ps();
for(; i<=(n-(8*nAcc)); i+=(8*nAcc)){
__m256 d[VEC1_L2_NACC];
for(c=0; c<nAcc; c++){
d[c] = _mm256_sub_ps(LOADU(&a1[i+c*8]), LOADU(&a2[i+c*8]));
acc[c] = FMADD(d[c], d[c], acc[c]);
}
}
/* Sum all nAcc accumulators into acc[0]. */
VERTICAL_SUM(acc, nAcc);
/* Process the part of the tail that can be done 8 elements at a time
** using acc[0] as the accumulator. Then sum all 8 elements of acc[0]
** into scalar float "ret". */
for(; i<=(n-8); i+=8){
__m256 d = _mm256_sub_ps(LOADU(&a1[i]), LOADU(&a2[i]));
acc[0] = FMADD(d, d, acc[0]);
}
HORIZONTAL_SUM(ret, acc[0]);
#endif /* VEC1_HAVE_AVX2 */
#ifdef VEC1_HAVE_NEON
# ifndef VEC1_L2_NACC
# define VEC1_L2_NACC 8
# endif
const int nAcc = VEC1_L2_NACC;
int c, s;
/* Create and initialize to zero nAcc accumulator registers. */
float32x4_t acc[VEC1_L2_NACC];
for(c=0; c<nAcc; c++) acc[c] = vdupq_n_f32(0.0f);
for(; i<=(n-(4*nAcc)); i+=(4*nAcc)){
float32x4_t d[VEC1_L2_NACC];
for(c=0; c<nAcc; c++){
d[c] = vsubq_f32(vld1q_f32(&a1[i+c*4]), vld1q_f32(&a2[i+c*4]));
acc[c] = vmlaq_f32(acc[c], d[c], d[c]);
}
}
/* Sum all nAcc accumulators into acc[0]. */
VERTICAL_SUM(acc, nAcc);
for(; i<=(n-4); i+=4){
float32x4_t d = vsubq_f32(vld1q_f32(&a1[i]), vld1q_f32(&a2[i]));
acc[0] = vmlaq_f32(acc[0], d, d);
}
ret = vaddvq_f32(acc[0]);
#endif /* VEC1_HAVE_NEON */
for(; i<n; i++){
double diff = a1[i] - a2[i];
ret += (diff * diff);
}
return ret;
}
/*
** Return the square of the norm for vector a1[n].
*/
static double vec1VectorNorm2(const float *a1, int n){
return vec1DotProduct(n, a1, a1);
}
/*
** Return the cosine distance between vectors a1 and a2, each n elements
** in size.
*/
static double vec1CosDist(
const float *a1,
const float *a2,
int n
){
int i = 0;
float dot = 0.0f;
float a1s = 0.0f;
float a2s = 0.0f;
#ifdef VEC1_HAVE_AVX2
__m256 va1s = _mm256_setzero_ps();
__m256 va2s = _mm256_setzero_ps();
__m256 vdot = _mm256_setzero_ps();
for(; i<=(n-8); i+=8){
__m256 va1 = _mm256_loadu_ps(&a1[i]);
__m256 va2 = _mm256_loadu_ps(&a2[i]);
va1s = FMADD(va1, va1, va1s);
va2s = FMADD(va2, va2, va2s);
vdot = FMADD(va1, va2, vdot);
}
HORIZONTAL_SUM(a1s, va1s);
HORIZONTAL_SUM(a2s, va2s);
HORIZONTAL_SUM(dot, vdot);
#endif /* VEC1_HAVE_AVX2 */
#ifdef VEC1_HAVE_NEON
float32x4_t va1s = vdupq_n_f32(0.0f);
float32x4_t va2s = vdupq_n_f32(0.0f);
float32x4_t vdot = vdupq_n_f32(0.0f);
for(; i<=(n-4); i+=4){
float32x4_t va1 = vld1q_f32(&a1[i]);
float32x4_t va2 = vld1q_f32(&a2[i]);
va1s = vmlaq_f32(va1s, va1, va1);
va2s = vmlaq_f32(va2s, va2, va2);
vdot = vmlaq_f32(vdot, va1, va2);
}
a1s = vaddvq_f32(va1s);
a2s = vaddvq_f32(va2s);
dot = vaddvq_f32(vdot);
#endif /* VEC1_HAVE_NEON */
for(; i<n; i++){
dot += a1[i] * a2[i];
a1s += a1[i] * a1[i];
a2s += a2[i] * a2[i];
}
/* If either vector was a zero vector, assume the two inputs are
** perfectly orthogonal. */
if( a1s==0.0 || a2s==0.0 ) return 1.0;
return 1.0 - (dot / sqrt(a1s * a2s));
}
/*
** This function must be called with two blobs:
**
** vec1_l2_distance(BLOB, BLOB)
**
** each of which are treated as vectors of type float in machine byte order.
** This function returns the L2 distance between the two vectors.
*/
static void vec1DistanceFuncL2(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
const float *a1 = (const float*)sqlite3_value_blob(aVal[0]);
const float *a2 = (const float*)sqlite3_value_blob(aVal[1]);
int n1 = sqlite3_value_bytes(aVal[0]);
int n2 = sqlite3_value_bytes(aVal[1]);
assert( nVal==2 );
UNUSED_PARAMETER(nVal);
if( n1!=n2 || (n1 % sizeof_f32)!=0
|| sqlite3_value_type(aVal[0])!=SQLITE_BLOB
|| sqlite3_value_type(aVal[1])!=SQLITE_BLOB
){
sqlite3_result_error(pCtx, "vec1_l2_distance: bad arguments", -1);
return;
}
sqlite3_result_double(pCtx, vec1L2Dist(a1, a2, n1 / sizeof_f32));
}
/*
** This function must be called with two blobs:
**
** vec1_cos_distance(BLOB, BLOB)
**
** each of which are treated as vectors of type float in machine byte order.
** This function returns the L2 distance between the two vectors.
*/
static void vec1DistanceFuncCos(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
const float *a1 = (const float*)sqlite3_value_blob(aVal[0]);
const float *a2 = (const float*)sqlite3_value_blob(aVal[1]);
int n1 = sqlite3_value_bytes(aVal[0]);
int n2 = sqlite3_value_bytes(aVal[1]);
assert( nVal==2 );
UNUSED_PARAMETER(nVal);
if( n1!=n2 || (n1 % sizeof_f32)!=0
|| sqlite3_value_type(aVal[0])!=SQLITE_BLOB
|| sqlite3_value_type(aVal[1])!=SQLITE_BLOB
){
sqlite3_result_error(pCtx, "vec1_cos_distance: bad arguments", -1);
return;
}
sqlite3_result_double(pCtx, vec1CosDist(a1, a2, n1 / sizeof_f32));
}
/*
** Allocate and return a zeroed buffer nByte bytes in size.
*/
static void *vec1MallocZero(i64 nByte){
void *pRet = sqlite3_malloc64(nByte);
if( pRet ){
memset(pRet, 0, nByte);
}
return pRet;
}
typedef struct Vec1Thread Vec1Thread;
typedef struct Vec1Mutex Vec1Mutex;
typedef struct Vec1Cond Vec1Cond;
/****************************************************************************
****************************************************************************/
/********************************* Unix Pthreads ****************************/
#if defined(VEC1_THREADS_PTHREADS)
#include <pthread.h>
/*
** Unix version of thread object.
*/
struct Vec1Thread {
pthread_t tid;
};
/*
** Unix version of mutex object.
*/
struct Vec1Mutex {
pthread_mutex_t mutex;
};
/*
** Unix version of condition variable.
*/
struct Vec1Cond {
pthread_cond_t cond;
};
/*
** Create a new thread running xTask(pIn). If successful, set *ppThread
** to the new thread handle and return SQLITE_OK. Otherwise, if an
** error occurs, set *ppThread to 0 and return either SQLITE_ERROR
** or SQLITE_NOMEM as appropriate.
*/
static int vec1ThreadCreate(
Vec1Thread **ppThread,
void *(*xTask)(void*),
void *pIn
){
Vec1Thread *p = sqlite3_malloc(sizeof(Vec1Thread));
if( p==0 ){ *ppThread = 0; return SQLITE_NOMEM; }
if( pthread_create(&p->tid, 0, xTask, pIn)!=0 ){
sqlite3_free(p);
*ppThread = 0;
return SQLITE_ERROR;
}
*ppThread = p;
return SQLITE_OK;
}
/*
** Join the thread pThread. If successful set *ppOut to the value
** returned by the thread's xTask routine and return SQLITE_OK.
** Otherwise, return SQLITE_ERROR and set *ppOut to 0.
**
** The thread handle may not be used after this function has been
** called. The results of doing so are undefined.
*/
static int vec1ThreadJoin(Vec1Thread *pThread, void **ppOut){
int rc = SQLITE_OK;
if( pthread_join(pThread->tid, ppOut)!=0 ){
rc = SQLITE_ERROR;
if( ppOut ) *ppOut = 0;
}
sqlite3_free(pThread);
return rc;
}
/*
** Create a new mutex. If successful, set *ppMutex to the new
** mutex handle and return SQLITE_OK. Otherwise, set *ppMutex to 0
** and return an SQLite error code - SQLITE_NOMEM or SQLITE_ERROR as
** appropriate.
*/
static int vec1MutexCreate(Vec1Mutex **ppMutex){
Vec1Mutex *p = sqlite3_malloc(sizeof(Vec1Mutex));
if( p==0 ){ *ppMutex = 0; return SQLITE_NOMEM; }
if( pthread_mutex_init(&p->mutex, 0)!=0 ){
sqlite3_free(p);
*ppMutex = 0;
return SQLITE_ERROR;
}
*ppMutex = p;
return SQLITE_OK;
}
/*
** Destroy (free) the mutex passed as the only argument.
*/
static void vec1MutexDestroy(Vec1Mutex *pMutex){
pthread_mutex_destroy(&pMutex->mutex);
sqlite3_free(pMutex);
}
/*
** Enter the mutex passed as the only argument. The results of calling
** this function on a mutex that is already held are undefined.
*/
static void vec1MutexEnter(Vec1Mutex *pMutex){
pthread_mutex_lock(&pMutex->mutex);
}
/*
** Leave the mutex passed as the only argument.
*/
static void vec1MutexLeave(Vec1Mutex *pMutex){
pthread_mutex_unlock(&pMutex->mutex);
}
/*
** Create a new posix style condition variable.
*/
static int vec1CondCreate(Vec1Cond **ppCond){
Vec1Cond *p = sqlite3_malloc(sizeof(Vec1Cond));
if( p==0 ){ *ppCond = 0; return SQLITE_NOMEM; }
if( pthread_cond_init(&p->cond, 0)!=0 ){
sqlite3_free(p);
*ppCond = 0;
return SQLITE_ERROR;
}
*ppCond = p;
return SQLITE_OK;
}
/*
** Destroy the condition variable passed as the only argument.
*/
static void vec1CondDestroy(Vec1Cond *pCond){
pthread_cond_destroy(&pCond->cond);
sqlite3_free(pCond);
}
/*
** Mutex pMutex must be held when this is called. It causes the
** thread to atomically relinquish the mutex and wait on the
** condition variable. When the condition is signaled by another
** thread calling vec1CondBroadcast(), this thread will reaquire
** mutex pMutex and continue.
*/
static void vec1CondWait(Vec1Mutex *pMutex, Vec1Cond *pCond){
pthread_cond_wait(&pCond->cond, &pMutex->mutex);
}
/*
** Wake up threads waiting on the condition variable passed as
** the only argument.
*/
static void vec1CondBroadcast(Vec1Cond *pCond){
pthread_cond_broadcast(&pCond->cond);
}
#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */
/******************************** End Unix Pthreads *************************/
/********************************* Win32 Threads ****************************/
#if defined(VEC1_THREADS_WINDOWS)
#include <windows.h>
struct Vec1Thread {
HANDLE hThread;
void *(*xTask)(void*);
void *pIn;
void *pOut;
};
struct Vec1Mutex {
CRITICAL_SECTION cs;
};
struct Vec1Cond {
CONDITION_VARIABLE cv;
};
/* Windows thread trampoline - WINAPI calling convention */
static DWORD WINAPI vec1ThreadMain(LPVOID pArg){
Vec1Thread *p = (Vec1Thread*)pArg;
p->pOut = p->xTask(p->pIn);
return 0;
}
static int vec1ThreadCreate(
Vec1Thread **ppThread,
void *(*xTask)(void*),
void *pIn
){
Vec1Thread *p = sqlite3_malloc(sizeof(Vec1Thread));
if( p==0 ){ *ppThread = 0; return SQLITE_NOMEM; }
p->xTask = xTask;
p->pIn = pIn;
p->pOut = 0;
p->hThread = CreateThread(NULL, 0, vec1ThreadMain, p, 0, NULL);
if( p->hThread==NULL ){
sqlite3_free(p);
*ppThread = 0;
return SQLITE_ERROR;
}
*ppThread = p;
return SQLITE_OK;
}
static int vec1ThreadJoin(Vec1Thread *pThread, void **ppOut){
int rc = SQLITE_OK;
if( WaitForSingleObject(pThread->hThread, INFINITE)!=WAIT_OBJECT_0 ){
rc = SQLITE_ERROR;
if( ppOut ) *ppOut = 0;
}else{
if( ppOut ) *ppOut = pThread->pOut;
}
CloseHandle(pThread->hThread);
sqlite3_free(pThread);
return rc;
}
static int vec1MutexCreate(Vec1Mutex **ppMutex){
Vec1Mutex *p = sqlite3_malloc(sizeof(Vec1Mutex));
if( p==0 ){ *ppMutex = 0; return SQLITE_NOMEM; }
InitializeCriticalSection(&p->cs);
*ppMutex = p;
return SQLITE_OK;
}
static void vec1MutexDestroy(Vec1Mutex *pMutex){
DeleteCriticalSection(&pMutex->cs);
sqlite3_free(pMutex);
}
static void vec1MutexEnter(Vec1Mutex *pMutex){
EnterCriticalSection(&pMutex->cs);
}
static void vec1MutexLeave(Vec1Mutex *pMutex){
LeaveCriticalSection(&pMutex->cs);
}
static int vec1CondCreate(Vec1Cond **ppCond){
Vec1Cond *p = sqlite3_malloc(sizeof(Vec1Cond));
if( p==0 ){ *ppCond = 0; return SQLITE_NOMEM; }
InitializeConditionVariable(&p->cv);
*ppCond = p;
return SQLITE_OK;
}
static void vec1CondDestroy(Vec1Cond *pCond){
/* Windows condition variables have no destroy function */
sqlite3_free(pCond);
}
static void vec1CondWait(Vec1Mutex *pMutex, Vec1Cond *pCond){
SleepConditionVariableCS(&pCond->cv, &pMutex->cs, INFINITE);
}
static void vec1CondBroadcast(Vec1Cond *pCond){
WakeAllConditionVariable(&pCond->cv);
}
#endif /* SQLITE_OS_WIN_THREADS */
/******************************** End Win32 Threads *************************/
/*
*****************************************************************************
*****************************************************************************/
typedef struct Vec1JobQueue Vec1JobQueue;
#if VEC1_THREADS
typedef struct Vec1Job Vec1Job;
struct Vec1Job {
void *pCtx;
void (*xWork)(void *pCtx);
int (*xFinish)(void *pCtx, int);
Vec1Job *pJobNext;
};
/*
** nJob is the number of jobs on either the pWaiting or pFinished list,
** or else being worked on by a thread. nJob is incremented when the
** job is added to pWaiting, decremented when it is removed from pFinished.
** Always by the main thread.
*/
struct Vec1JobQueue {
Vec1Mutex *pMutex;
Vec1Cond *pCond1;
Vec1Cond *pCond2;
Vec1Thread **apWorker;
int nWorker; /* Number of worker threads */
int bDone; /* True if threads should exit */
int nJob; /* Total number of unfinished jobs */
Vec1Job *pWaiting; /* List of jobs waiting for xWork() */
Vec1Job *pFinished; /* List of jobs waiting for xFinish() */
Vec1Job *pFree; /* List of spare Vec1Job structures */
};
static void *vec1JobWorkerMain(void *pCtx){
Vec1JobQueue *p = (Vec1JobQueue*)pCtx;
while( 1 ){
Vec1Job *pJob = 0;
/* Try to find a new job to work on. */
vec1MutexEnter(p->pMutex);
while( p->pWaiting==0 && p->bDone==0 ){
vec1CondWait(p->pMutex, p->pCond1);
}
pJob = p->pWaiting;
if( pJob ) p->pWaiting = pJob->pJobNext;
vec1MutexLeave(p->pMutex);
/* If pJob is 0, we're done. Thread should exit. */
if( pJob==0 ) break;
/* Do the work for this job. */
pJob->xWork(pJob->pCtx);
/* Put the job on the finished list. If it is currently empty, the
** main thread may be waiting. Signal it in that case. */
vec1MutexEnter(p->pMutex);
if( p->pFinished==0 ) vec1CondBroadcast(p->pCond2);
pJob->pJobNext = p->pFinished;
p->pFinished = pJob;
vec1MutexLeave(p->pMutex);
}
return 0;
}
static int vec1JobQueueFinishJobs(Vec1JobQueue *pQueue, int rcin){
int rc = rcin;
if( pQueue==0 ) return rc;
while( 1 ){
Vec1Job *pFin = 0;
Vec1Job *pWork = 0;
/* Try to find a job to help with. If there is nothing, wait on pCond2. */
vec1MutexEnter(pQueue->pMutex);
while( pQueue->pFinished==0 && pQueue->pWaiting==0 && pQueue->nJob>0 ){
vec1CondWait(pQueue->pMutex, pQueue->pCond2);
}
if( pQueue->pFinished ){
pFin = pQueue->pFinished;
pQueue->pFinished = pFin->pJobNext;
pQueue->nJob--;
}else if( pQueue->pWaiting ){
pWork = pQueue->pWaiting;
pQueue->pWaiting = pWork->pJobNext;
pQueue->nJob--;
}
vec1MutexLeave(pQueue->pMutex);
if( pFin==0 && pWork==0 ) break;
if( pWork ){
pWork->xWork(pWork->pCtx);
pFin = pWork;
}
{
int rc2 = pFin->xFinish(pFin->pCtx, rc);
if( rc==SQLITE_OK ) rc = rc2;
}
vec1MutexEnter(pQueue->pMutex);
pFin->pJobNext = pQueue->pFree;
pQueue->pFree = pFin;
vec1MutexLeave(pQueue->pMutex);
}
return rc;
}
static void vec1JobQueueFree(Vec1JobQueue *pQueue){
if( pQueue ){
if( pQueue->pMutex && pQueue->pCond1 ){
int ii;
vec1JobQueueFinishJobs(pQueue, SQLITE_ERROR);
vec1MutexEnter(pQueue->pMutex);
pQueue->bDone = 1;
vec1CondBroadcast(pQueue->pCond1);
vec1MutexLeave(pQueue->pMutex);
for(ii=0; ii<pQueue->nWorker; ii++){
void *dummy = 0;
vec1ThreadJoin(pQueue->apWorker[ii], &dummy);
}
}
if( pQueue->pMutex ) vec1MutexDestroy(pQueue->pMutex);
if( pQueue->pCond1 ) vec1CondDestroy(pQueue->pCond1);
if( pQueue->pCond2 ) vec1CondDestroy(pQueue->pCond2);
assert( pQueue->pWaiting==0 );
assert( pQueue->pFinished==0 );
assert( pQueue->pFree!=0 );
sqlite3_free(pQueue);
}
}
/*
** Allocate a new job-queue.
*/
static Vec1JobQueue *vec1JobQueueNew(int nWorker){
int rc = SQLITE_OK;
Vec1JobQueue *pNew = 0;
int nJob = nWorker + 8;
int nByte =
sizeof(Vec1JobQueue) +
sizeof(Vec1Thread*)*nWorker +
sizeof(Vec1Job) * nJob;
assert( nWorker>=0 );
pNew = (Vec1JobQueue*)vec1MallocZero(nByte);
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int ii;
Vec1Job *pJob;
pNew->apWorker = (Vec1Thread**)&pNew[1];
pJob = (Vec1Job*)&pNew->apWorker[nWorker];
for(ii=0; ii<nJob; ii++){
pJob->pJobNext = pNew->pFree;
pNew->pFree = pJob;
pJob++;
}
rc = vec1MutexCreate(&pNew->pMutex);
if( rc==SQLITE_OK ) rc = vec1CondCreate(&pNew->pCond1);
if( rc==SQLITE_OK ) rc = vec1CondCreate(&pNew->pCond2);
for(ii=0; rc==SQLITE_OK && ii<nWorker; ii++){
rc = vec1ThreadCreate(&pNew->apWorker[ii], vec1JobWorkerMain, pNew);
if( rc==SQLITE_OK ){
pNew->nWorker++;
}
}
}
if( rc!=SQLITE_OK ){
vec1JobQueueFree(pNew);
pNew = 0;
}
return pNew;
}
static int vec1DummyFinal(void *pCtx, int rcin){
UNUSED_PARAMETER(pCtx);
return rcin;
}
static int vec1JobQueueAddJob(
Vec1JobQueue *pQueue,
void (*xWork)(void *pCtx),
int (*xFinish)(void *pCtx, int),
void *pCtx
){
Vec1Job *pFree = 0;
Vec1Job *pFinish = 0;
Vec1Job *pWork = 0;
int rc = SQLITE_OK;
if( xFinish==0 ) xFinish = vec1DummyFinal;
if( pQueue==0 ){
xWork(pCtx);
return xFinish(pCtx, SQLITE_OK);
}
/* Check if this thread needs to do any xFinish() or xWork() calls. And
** try to get a free Vec1Job object at the same time. */
vec1MutexEnter(pQueue->pMutex);
assert( pQueue->pWaiting || pQueue->pFree || pQueue->pFinished );
if( pQueue->pFree ){
pFree = pQueue->pFree;
pQueue->pFree = pFree->pJobNext;
}else if( pQueue->pFinished ){
pFinish = pQueue->pFinished;
pQueue->pFinished = pFinish->pJobNext;
pQueue->nJob--;
}else{
pWork = pQueue->pWaiting;
pQueue->pWaiting = pWork->pJobNext;
pQueue->nJob--;
}
vec1MutexLeave(pQueue->pMutex);
/* At most 1 of pFree, pFinish and pWork may be non NULL */
assert( (pFree!=0)+(pFinish!=0)+(pWork!=0)<=1 );
if( pWork ){
pWork->xWork(pWork->pCtx);
pFinish = pWork;
}
if( pFinish ){
rc = pFinish->xFinish(pFinish->pCtx, SQLITE_OK);
pFree = pFinish;
}
assert( pFree );
pFree->pCtx = pCtx;
pFree->xWork = xWork;
pFree->xFinish = xFinish;
vec1MutexEnter(pQueue->pMutex);
if( pQueue->pWaiting==0 ) vec1CondBroadcast(pQueue->pCond1);
pFree->pJobNext = pQueue->pWaiting;
pQueue->pWaiting = pFree;
pQueue->nJob++;
vec1MutexLeave(pQueue->pMutex);
return rc;
}
#else /* VEC1_THREADS=0 */
/*
** Dummy implementation of job-queue for single-threaded builds.
*/
static int vec1JobQueueFinishJobs(Vec1JobQueue *pQueue, int rcin){
return rcin;
}
static void vec1JobQueueFree(Vec1JobQueue *pQueue){}
static Vec1JobQueue *vec1JobQueueNew(int a){ return 0; }
static int vec1JobQueueAddJob(
Vec1JobQueue *pQueue,
void (*xWork)(void *pCtx),
int (*xFinish)(void *pCtx, int),
void *pCtx
){
xWork(pCtx);
return xFinish ? xFinish(pCtx, SQLITE_OK) : SQLITE_OK;
}
#endif
/*
** Minimum and maximum values allowed for the 'blocksize' command.
*/
#define VEC1_BLOCKSIZE_MIN 1024
#define VEC1_BLOCKSIZE_MAX (16*1024*1024)
#define VEC1_BLOCKSIZE_DEFAULT (1024*1024)
#define VEC1_BLOCKSIZE_MIN_DEFAULT (16*1024)
#define VEC1_VECSIZE_MIN 2
#define VEC1_VECSIZE_MAX 4096
/*
** Number of codes in each codebook. This is always 256.
*/
#define VEC1_PQ_CODEBOOK_SZ 256
#define VEC1_MIN_CODESIZE 1
#define VEC1_MAX_CODESIZE 256
#define VEC1_MIN_BUCKET 0
#define VEC1_MAX_BUCKET 65536
#define VEC1_MIN_NOPQ_ROUND 1
#define VEC1_MAX_NOPQ_ROUND 32
#define VEC1_PQ_KMEANS_NITER 30
#define VEC1_PQ_KMEANS_STOPVAL 0.999
/*
** Minimum number of vectors required for PQ training.
*/
#define VEC1_PQ_TRAINING_SET_MIN (VEC1_PQ_CODEBOOK_SZ * 2)
#define VEC1_TRAINING_TOTAL 0
#define VEC1_TRAINING_ROT_CODEBOOKS 1
#define VEC1_TRAINING_ROT_COVARIANCE 2
#define VEC1_TRAINING_ROT_JACOBI 3
#define VEC1_TRAINING_ROT_VECTORS 4
#define VEC1_TRAINING_COARSE_INIT 5
#define VEC1_TRAINING_COARSE_KMEANS 6
#define VEC1_TRAINING_RESIDUALS 7
#define VEC1_TRAINING_CODEBOOKS 8
#define VEC1_TRAINING_NTIMER 9
/*
** When accumulating training data, allocate space in chunks of
** VEC1_TRAINING_SZCHUNK. Allow at most VEC1_TRAINING_MAXCHUNK chunks of
** training data. With default values, the limit is 64GiB of training
** vectors.
*/
#define VEC1_TRAINING_MAXCHUNK 1024
#define VEC1_TRAINING_SZCHUNK (32*1024*1024)
/*
** Values for Vec1TrainCtx.eQuant
*/
#define VEC1_QUANTIZE_NONE 0
#define VEC1_QUANTIZE_PQ 1
#define VEC1_QUANTIZE_OPQ 2
#define VEC1_QUANTIZE_BQ 3
/*
** Training vectors collected by xStep() are stored in an instance
** of this structure.
*/
typedef struct Vec1TrainVectors Vec1TrainVectors;
struct Vec1TrainVectors {
int nElem; /* Number of elements in each vector */
int nVec; /* Number of vectors in chunk array */
int nChunk; /* Number of chunks allocated so far */
int nVecPerChunk; /* Number of vectors in each chunk */
float *aChunk[VEC1_TRAINING_MAXCHUNK];
};
/*
** Context object for the vec1_train() aggregate function.
**
** eQuant:
** A VEC1_QUANTIZE_XYZ value. Set by the "quantizer" option.
**
** nCodesize:
** For PQ or OPQ, the number of vector sub-spaces. For BQ, the number of
** bytes required to store the bit-vector. For 1 bit per dimension this
** value is set to ((d+7)/8).
**
** nBucket:
** Number of IVF buckets used by coarse quantizer. Set to 0 to disable
** coarse quant altogether.
*/
typedef struct Vec1TrainCtx Vec1TrainCtx;
struct Vec1TrainCtx {
/* Training vectors collected by xStep() */
Vec1TrainVectors tv;
/* Training parameters configured by json argument */
int eDistance; /* VEC1_DISTANCE_L2 or COS */
int eQuant; /* VEC1_QUANTIZE_XX value */
int nCodesize; /* Number of vector sub-spaces */
int nBucket; /* Number of IVF buckets */
int bOpq;
int bResidual;
int bSvdVerify;
int bWHT; /* Use WHT transformation */
int nThread; /* Number of CPU threads to use */
int nOpqRound; /* Number of OPQ rounds */
char *zLogFunction; /* Name of user SQL function for logging */
char *zProfileFunction; /* Name of user SQL function for profile */
sqlite3_stmt *pLogStmt; /* SELECT zLogFunction(?, ?) */
sqlite3_context *pCtx; /* Context object for errors */
int rc; /* Error code for training operation */
/* Used for calculating "percent" value passed to "SELECT zLogFunction()" */
int nTotalWork;
int nCompletedWork;
/* Calculated based on training parameters + size of first vector */
int nCodeElem; /* Number of elements in codebook vectors */
u64 aTime[VEC1_TRAINING_NTIMER];
};
#define VEC1_TRAINING_WORK_OPQ 2
#define VEC1_TRAINING_WORK_COARSE1 1
#define VEC1_TRAINING_WORK_COARSE2 1
#define VEC1_TRAINING_WORK_CODEBOOKS 1
/*
** Return a pointer to the requested training vector. The results are
** undefined if parameter iVec is out of range.
*/
static float *vec1TrainingVector(Vec1TrainVectors *p, int iVec){
int iChunk = iVec / p->nVecPerChunk;
int iOff = (iVec % p->nVecPerChunk) * p->nElem;
return &((p->aChunk[iChunk])[iOff]);
}
/*
** aSub[] is a vector of nCodeElem elements. aCodebook[] is a codebook of
** nCode vectors, each of nCodeElem elements. Return the index of the best
** match for aSub in aCodebook[].
*/
static int vec1BestMatchSimple(
const float *aCodebook, /* Codebook of nCodeElem element vectors */
int nCode, /* Number of vectors in aCodebook */
const float *aSub, /* nCodeElem element vector */
int nCodeElem, /* Size of vectors in elements */
double *pfBest /* Distance to best match */
){
double fBestDist = INFINITY;
int iBest = 0;
int ii;
for(ii=0; ii<nCode; ii++){
double fDist = vec1L2Dist(aSub, &aCodebook[ii*nCodeElem], nCodeElem);
if( fDist<fBestDist ){
iBest = ii;
fBestDist = fDist;
}
}
assert( iBest>=0 && iBest<nCode );
if( pfBest ) *pfBest += fBestDist;
return iBest;
}
/*
** Relative tolerance used by vec1BucketIsNearTie().
*/
#define VEC1_BUCKET_TOLERANCE 0.00001
/*
** fBest is the distance from a vector to its nearest IVF centroid, as
** found by vec1BestMatchSimple(). fDist is the distance from the same
** vector to some other centroid. Return true if the two distances are
** close enough that rounding differences could account for the difference.
*/
static int vec1BucketIsNearTie(double fBest, double fDist){
return (fDist - fBest) <= (fBest * VEC1_BUCKET_TOLERANCE);
}
/*
** Return a pseudo-random positive 32-bit value.
*/
static int vec1Rand31(){
int val;
sqlite3_randomness(sizeof(val), (void*)&val);
return (val & 0x7FFFFFFF);
}
/*
** Access nElem element vector iVec from packed array aPacked.
*/
#define PACKEDVEC(aPacked, nElem, iVec) (&(aPacked)[(iVec)*(nElem)])
/*
** Access training vector i from Vec1TrainCtx p.
*/
#define TRAININGVEC(p, i) PACKEDVEC(p->aVec, p->nElem, i)
typedef struct LloydsJob LloydsJob;
struct LloydsJob {
/* Data to train on */
Vec1TrainVectors *pVec;
int iFirst; /* First vector for this job */
int iEof; /* Index of last vector +1 for this job */
int nK; /* Number of centroids */
const float *aCentroid; /* Centroids */
#ifdef VEC1_USE_TRANSPOSITION
float *aTrans; /* Transposed centroids */
#endif
/* Outputs */
double fTotal;
float *aSum;
int *aCount;
};
/*
** Context object for simple PRNG. See functions vec1RandomInit() and
** vec1RandomNext() for details.
*/
typedef struct Vec1Random Vec1Random;
struct Vec1Random {
u64 state;
u64 inc;
};
static u32 vec1RandomNext(Vec1Random *p) {
u32 xorshifted, rot;
u64 old = p->state;
p->state = old * 6364136223846793005ULL + p->inc;
xorshifted = (u32)(((old >> 18u) ^ old) >> 27u);
rot = (u32)(old >> 59u);
return ((xorshifted >> rot) | (xorshifted << ((32 - rot) & 31))) & 0x7FFFFFFF;
}
static void vec1RandomInit(Vec1Random *p, u64 seed, u64 id){
p->state = 0;
p->inc = (id << 1u) | 1u;
vec1RandomNext(p);
p->state += seed;
vec1RandomNext(p);
}
typedef struct KMeansInitJob KMeansInitJob;
struct KMeansInitJob {
int iFirst; /* First vector for this job */
int iEof; /* Index of last vector +1 for this job */
double fTotalMin; /* Total of aMin[iFirst..iEof-1] */
Vec1TrainVectors *pVec;
u32 *aWeight;
float *aMin;
int *aBest;
float *aCentroid;
int nCentroid;
int iFirstNew;
};
static void vec1KMeansInitWork(void *pCtx){
KMeansInitJob *p = (KMeansInitJob*)pCtx;
const int nElem = p->pVec->nElem;
double fTotalMin = 0.0;
int ii;
for(ii=p->iFirst; ii<p->iEof; ii++){
int jj;
float *vec = vec1TrainingVector(p->pVec, ii);
for(jj=p->iFirstNew; jj<p->nCentroid; jj++){
float *cent = &p->aCentroid[jj * nElem];
float dist = (float)vec1L2Dist(vec, cent, nElem);
if( jj==0 || dist<p->aMin[ii] ){
p->aMin[ii] = dist;
if( p->aBest ) p->aBest[ii] = jj;
}
}
fTotalMin += ((double)p->aMin[ii] * (p->aWeight ? p->aWeight[ii] : 1));
}
p->fTotalMin = fTotalMin;
}
/*
** Macro used to assign vectors to jobs in cases where an algorithm
** parallelizes across vectors. The pJob object must have at least the
** following member:
**
** struct XyzJob {
** ...
** Vec1TrainVectors *pVec; // Training vectors to use a subset of
** int iFirst; // index of first vector in pVec to use
** int iEof; // 1 past index of last vector in pVec to use
** ...
** };
*/
#define ASSIGN_VECTORS_TO_JOB(pV, pJob, iJob, nJob) \
pJob->pVec = (pV); \
pJob->iFirst = (iJob==0) ? 0 : pJob[-1].iEof; \
pJob->iEof = pJob->iFirst + ((pV)->nVec - pJob->iFirst) / (nJob - iJob); \
assert( pJob->iEof<=(pV)->nVec ); \
assert( (iJob==nJob-1)==(pJob->iEof==(pV)->nVec) );
/*
**
*/
static int vec1KMeansInitOne(
Vec1TrainVectors *pVec,
u32 *aWeight,
Vec1JobQueue *pQueue, /* Job queue to use (NULL -> 1 thread) */
int nK, /* Required number of outputs */
int nPerRound, /* Number of centroids per round */
int nTail, /* Number of tail rounds */
float *aOut, /* Array to populate with centroids */
u32 *aHist /* Array to populate with histogram */
){
const int nJob = (pQueue ? pQueue->nWorker + 1 : 1);
const int nElem = pVec->nElem;
const int nVByte = nElem * sizeof_f32;
Vec1Random rnd;
int nByte; /* Bytes of space to allocate */
KMeansInitJob *aJob = 0;
float *aMin = 0;
int *aBest = 0;
int ii; /* Loop counter for per-job loop */
int rr; /* Loop counter for per-round loop */
int iNext;
int nCentroid = 0;
int nNewCentroid = 0;
float *vec = 0;
vec1RandomInit(&rnd, vec1Rand31(), 0);
nByte =
(sizeof(KMeansInitJob) * nJob) +
(pVec->nVec * sizeof_f32) +
(aHist ? pVec->nVec * sizeof_u32 : 0);
aJob = vec1MallocZero(nByte);
if( aJob==0 ){
return SQLITE_NOMEM;
}
aMin = (float*)&aJob[nJob];
if( aHist ){
aBest = (int*)&aMin[pVec->nVec];
}
for(ii=0, iNext=0; ii<nJob; ii++){
KMeansInitJob *pJob = &aJob[ii];
ASSIGN_VECTORS_TO_JOB(pVec, pJob, ii, nJob);
pJob->aMin = aMin;
pJob->aBest = aBest;
pJob->aWeight = aWeight;
pJob->aCentroid = aOut;
}
/* First centroid is allocated by random selection. */
vec = vec1TrainingVector(pVec, vec1RandomNext(&rnd)%pVec->nVec);
memcpy(aOut, vec, nVByte);
nCentroid = nNewCentroid = 1;
for(rr=1; rr<nK; rr+=nNewCentroid){
int vv;
double fTotalMin = 0.0;
/* Step 2: */
for(ii=0; ii<nJob; ii++){
aJob[ii].nCentroid = nCentroid;
aJob[ii].iFirstNew = nCentroid - nNewCentroid;
vec1JobQueueAddJob(pQueue, vec1KMeansInitWork, 0, (void*)&aJob[ii]);
}
vec1JobQueueFinishJobs(pQueue, SQLITE_OK);
for(ii=0; ii<nJob; ii++){
fTotalMin += aJob[ii].fTotalMin;
}
/* Determine number of centroids to add: */
if( (nK-nCentroid)<=nTail ){
nNewCentroid = 1;
}else{
nNewCentroid = MIN(nPerRound, nK-nTail-nCentroid);
}
assert( nCentroid+nNewCentroid<=nK );
/* Choose new centroids */
if( nNewCentroid==1 ){
/* KMeans ++ case */
float fTarget = (float)((fTotalMin * vec1RandomNext(&rnd)) / 0x7FFFFFFF);
for(vv=0; vv<pVec->nVec-1; vv++){
fTarget -= (aMin[vv] * (aWeight ? aWeight[vv] : 1));
if( fTarget<=0.0 ) break;
}
memcpy(&aOut[nCentroid*nElem], vec1TrainingVector(pVec, vv), nVByte);
nCentroid++;
}else{
/* KMeans || case */
int cc;
vv = vec1RandomNext(&rnd) % pVec->nVec;
for(cc=0; cc<nNewCentroid; cc++){
while( 1 ){
u32 iRnd = vec1RandomNext(&rnd);
u32 w = (aWeight ? aWeight[vv] : 1);
double fRnd = ((double)iRnd * fTotalMin) / (double)0x7FFFFFFF;
if( fRnd<=((double)aMin[vv] * nNewCentroid * w) ) break;
vv = (vv + 1) % pVec->nVec;
}
memcpy(&aOut[nCentroid*nElem], vec1TrainingVector(pVec, vv), nVByte);
nCentroid++;
}
}
}
if( aHist ){
int vv;
for(vv=0; vv<pVec->nVec; vv++){
aHist[aBest[vv]]++;
}
}
sqlite3_free(aJob);
return SQLITE_OK;
}
/*
** Implementation of KMeans || initialization.
*/
static int vec1KMeansParallel(
Vec1TrainVectors *pVec,
Vec1JobQueue *pQueue, /* Job queue to use (NULL -> 1 thread) */
int nK, /* Required number of outputs */
float *aOut /* Array to populate with centroids */
){
int rc = SQLITE_OK;
const int nCent = nK * 2;
const int nByte = nCent * (pVec->nElem * sizeof_f32 + sizeof_u32);
float *aCent;
aCent = (float*)vec1MallocZero(nByte);
if( aCent==0 ){
rc = SQLITE_NOMEM;
}else{
u32 *aWeight = (u32*)&aCent[pVec->nElem * nCent];
rc = vec1KMeansInitOne(pVec, 0, pQueue, nCent, nCent/10, 0, aCent,aWeight);
if( rc==SQLITE_OK ){
Vec1TrainVectors vv;
memset(&vv, 0, sizeof(vv));
vv.nElem = pVec->nElem;
vv.nVec = nCent;
vv.nVecPerChunk = nCent;
vv.nChunk = 1;
vv.aChunk[0] = aCent;
rc = vec1KMeansInitOne(&vv, aWeight, pQueue, nK, nK, 0, aOut, 0);
}
sqlite3_free(aCent);
}
return rc;
}
static int vec1KMeansPlusPlus(
Vec1TrainVectors *pVec,
Vec1JobQueue *pQueue, /* Job queue to use (NULL -> 1 thread) */
int nK, /* Required number of outputs */
float *aOut /* Array to populate with centroids */
){
const int nPerRound = 8;
const int nTailRound = 32;
return vec1KMeansInitOne(pVec, 0, pQueue, nK, nPerRound, nTailRound, aOut, 0);
}
#if defined(_MSC_VER)
# define VEC1_NOINLINE __declspec(noinline)
#else
# define VEC1_NOINLINE __attribute__((noinline))
#endif
static VEC1_NOINLINE void vec1AddInPlaceScalar(
float * a1,
const float * a2,
int n
){
int i;
for(i=0; i<n; i++){
a1[i] += a2[i];
}
}
/*
** a1[] and a2[] are both vectors n elements in size. Add a2 to a1:
**
** a1 = a1 + a2
*/
static void vec1AddInPlace(float * a1, const float * a2, int n){
int i = 0;
#ifdef VEC1_HAVE_AVX2
for(; i<=n-8; i+=8){
__m256 v1 = _mm256_loadu_ps(a1 + i);
__m256 v2 = _mm256_loadu_ps(a2 + i);
_mm256_storeu_ps(a1 + i, _mm256_add_ps(v1, v2));
}
#endif
#ifdef VEC1_HAVE_NEON
for(; i<=n-4; i+=4){
float32x4_t v1 = vld1q_f32(&a1[i]);
float32x4_t v2 = vld1q_f32(&a2[i]);
vst1q_f32(&a1[i], vaddq_f32(v1, v2));
}
#endif
if( i!=n ) vec1AddInPlaceScalar(&a1[i], &a2[i], n-i);
}
#ifdef VEC1_HAVE_NEON
static inline void vec1MultiBestMatch(
int nElem, /* Size of vectors in elements */
int iCentroid, /* Centroid offset */
const float *aVec[VEC1_MULTIMATCH_NVEC],
const float *aCent, /* VEC1_MULTIMATCH_NCENT packed centroids */
float *aDistIn,
int *aBestIn
){
float32x4_t aAcc[VEC1_MULTIMATCH_NVEC][VEC1_MULTIMATCH_NCENT];
int vv; /* 0 to VEC1_MULTIMATCH_NVEC */
int cc; /* 0 to VEC1_MULTIMATCH_NCENT */
int d; /* 0 to nElem */
float aDist[VEC1_MULTIMATCH_NVEC];
int aBest[VEC1_MULTIMATCH_NVEC];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
for(cc=0; cc<VEC1_MULTIMATCH_NCENT; cc++){
aAcc[vv][cc] = vdupq_n_f32(0.0f);
}
}
for(d=0; d<=(nElem-4); d+=4){
float32x4_t aV[VEC1_MULTIMATCH_NVEC];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aV[vv] = vld1q_f32(&aVec[vv][d]);
}
for(cc=0; cc<VEC1_MULTIMATCH_NCENT; cc++){
float32x4_t cent = vld1q_f32(&aCent[cc*nElem+d]);
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float32x4_t diff = vsubq_f32(aV[vv], cent);
aAcc[vv][cc] = vfmaq_f32(aAcc[vv][cc], diff, diff);
}
}
}
/* Scalar tail for cases where the number of dimensions is not
** divisible by 4. */
for(; d<nElem; d++){
for(cc=0; cc<VEC1_MULTIMATCH_NCENT; cc++){
float cent = aCent[cc*nElem+d];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float diff = cent - aVec[vv][d];
aAcc[vv][cc] = vsetq_lane_f32(
vgetq_lane_f32(aAcc[vv][cc], 0) + (diff*diff), aAcc[vv][cc], 0
);
}
}
}
#if 0
{
float32x4_t vBestDist = vld1q_f32(aDist);
int32x4_t vBestIdx = vld1q_s32(aBest);
for(cc=0; cc<VEC1_MULTIMATCH_NCENT; cc++){
float32x4_t cand = vdupq_n_f32(0.0f);
uint32x4_t mask;
cand = vsetq_lane_f32(vaddvq_f32(aAcc[0][cc]), cand, 0);
cand = vsetq_lane_f32(vaddvq_f32(aAcc[1][cc]), cand, 1);
cand = vsetq_lane_f32(vaddvq_f32(aAcc[2][cc]), cand, 2);
cand = vsetq_lane_f32(vaddvq_f32(aAcc[3][cc]), cand, 3);
mask = vcltq_f32(cand, vBestDist);
vBestDist = vbslq_f32(mask, cand, vBestDist);
vBestIdx = vbslq_s32(mask, vdupq_n_s32(iCentroid+cc), vBestIdx);
}
vst1q_f32(aDist, vBestDist);
vst1q_s32(aBest, vBestIdx);
}
#endif
memcpy(aDist, aDistIn, sizeof_f32*VEC1_MULTIMATCH_NVEC);
memcpy(aBest, aBestIn, sizeof_u32*VEC1_MULTIMATCH_NVEC);
for(cc=0; cc<VEC1_MULTIMATCH_NCENT; cc++){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float dist = vaddvq_f32(aAcc[vv][cc]);
if( dist<aDist[vv] ){
aDist[vv] = dist;
aBest[vv] = iCentroid + cc;
}
}
}
memcpy(aDistIn, aDist, sizeof_f32*VEC1_MULTIMATCH_NVEC);
memcpy(aBestIn, aBest, sizeof_u32*VEC1_MULTIMATCH_NVEC);
}
#endif
/*
** Find the best match for each of VEC1_MULTIMATCH_NVEC vectors from the
** nCent centroids encoded in array aCentNT. The difference between this
** function and vec1BestMatchN() is the format of aCentNT. For this
** function, centroids are arranged into blocks of VEC1_NTRANSPOSE_WIDTH
** centroids per block and transposed. e.g. if VEC1_NTRANSPOSE_WIDTH==8,
** aCent begins with:
**
** * d0 value for each of the first 8 vectors, followed by
** * d1 value for each of the first 8 vectors, followed by
** ...
** * d(nElem-1) value for each of the first 8 vectors, then
** * d0 value for each of the second 8 vectors, etc.
** ...
*/
#ifdef VEC1_USE_TRANSPOSITION
static void vec1BestMatchTransposeN(
int nElem,
const float *aCent,
const float *aCentNT,
int nCent,
const float *aVec[VEC1_MULTIMATCH_NVEC],
int aBestOut[VEC1_MULTIMATCH_NVEC],
double *pfTotalOut
){
const int nWidth = VEC1_NTRANSPOSE_WIDTH;
float aDist[VEC1_MULTIMATCH_NVEC];
int aBest[VEC1_MULTIMATCH_NVEC];
int vv; /* For looping 0..VEC1_MULTIMATCH_NVEC-1 */
int iCent;
const float *pTrans = aCentNT;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aDist[vv] = INFINITY;
}
/* All the AVX2 code below assumes VEC1_NTRANSPOSE_WIDTH==8 */
assert( nWidth==8 );
for(iCent=0; iCent<=(nCent-nWidth); iCent+=nWidth){
__m256 aAcc[VEC1_MULTIMATCH_NVEC];
int d;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aAcc[vv] = _mm256_setzero_ps();
}
for(d=0; d<nElem; d++){
__m256 cval = LOADU( pTrans );
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
__m256 sval = _mm256_set1_ps( aVec[vv][d] );
__m256 diff = _mm256_sub_ps( cval, sval );
aAcc[vv] = FMADD(diff, diff, aAcc[vv]);
}
pTrans += nWidth;
}
/* At this point, each register in aAcc[] contains the distances
** from the corresponding training vector (vector ii + vv) to
** centroids iCent to (iCent + VEC1_MULTIMATCH_NCENT). Populate the
** aDist[] and aBest[] scalar arrays accordingly. */
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
__m256 vMin = _mm256_set1_ps(aDist[vv]);
__m256 mask = _mm256_cmp_ps(aAcc[vv], vMin, _CMP_LT_OQ);
if( _mm256_movemask_ps(mask)!=0 ){
/* This branch runs if aAcc[vv] contains one of more values less
** than scalar value aDist[vv]. */
int is;
float aScalar[VEC1_NTRANSPOSE_WIDTH];
_mm256_storeu_ps(aScalar, aAcc[vv]);
for(is=0; is<VEC1_NTRANSPOSE_WIDTH; is++){
if( aScalar[is]<aDist[vv] ){
aDist[vv] = aScalar[is];
aBest[vv] = is + iCent;
}
}
}
}
}
for( ; iCent<nCent; iCent++){
const float *cent = &aCent[iCent*nElem];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float dist = (float)vec1L2Dist(aVec[vv], cent, nElem);
if( dist<aDist[vv] ){
aDist[vv] = dist;
aBest[vv] = iCent;
}
}
}
memcpy(aBestOut, aBest, sizeof(aBest));
if( pfTotalOut ){
double total = 0.0;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
total += aDist[vv];
}
*pfTotalOut += total;
}
}
#define vec1BestMatchPlatformN(nElem, aCent, aTrans, nCent, aVec, aBest, p) \
vec1BestMatchTransposeN(nElem, aCent, aTrans, nCent, aVec, aBest, p)
#else
/*
** Array aCent[] contains nCent centroid vectors, each nElem elements in
** length. Array aVec[] contains VEC1_MULTIMATCH_NVEC vectors, also each
** nElem elements in length. This function finds the best match among
** the centroids for each of the VEC1_MULTIMATCH_NVEC vectors, and populates
** output array aBest with the index of each. Additionally, if it is not
** NULL, then (*pfTotalDist) is incremented by the total distance between
** each vector and it's nearest centroid.
*/
static void vec1BestMatchN(
int nElem, /* Number of elements in vectors */
const float *aCent, /* nCent packed centroids */
int nCent, /* Number of centroids */
const float *aVec[VEC1_MULTIMATCH_NVEC],
int aBest[VEC1_MULTIMATCH_NVEC],
double *pfTotalDist /* IN/OUT: Increment by total distortion */
){
int cc;
float aDist[VEC1_MULTIMATCH_NVEC];
for(cc=0; cc<VEC1_MULTIMATCH_NVEC; cc++){
aDist[cc] = INFINITY;
}
for(cc=0; cc<=(nCent-VEC1_MULTIMATCH_NCENT); cc+=VEC1_MULTIMATCH_NCENT){
#if defined(VEC1_HAVE_NEON)
vec1MultiBestMatch(nElem, cc, aVec, &aCent[cc*nElem], aDist, aBest);
#else
int ii;
int vv;
for(ii=0; ii<VEC1_MULTIMATCH_NCENT; ii++){
const float *cent = &aCent[(ii+cc)*nElem];
for(vv=0; vv<VEC1_MULTIMATCH_NCENT; vv++){
float dist = (float)vec1L2Dist(cent, aVec[vv], nElem);
if( dist<aDist[vv] ){
aDist[vv] = dist;
aBest[vv] = ii+cc;
}
}
}
#endif
}
/* Scalar tail in case the number of centroids is not an integer multiple
** of VEC1_MULTIMATCH_NCENT. */
for(; cc<nCent; cc++){
int vv;
const float *cent = &aCent[cc*nElem];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float dist = (float)vec1L2Dist(aVec[vv], cent, nElem);
if( dist<aDist[vv] ){
aDist[vv] = dist;
aBest[vv] = cc;
}
}
}
if( pfTotalDist ){
double fDist = *pfTotalDist;
for(cc=0; cc<VEC1_MULTIMATCH_NVEC; cc++){
fDist += aDist[cc];
}
*pfTotalDist = fDist;
}
}
/*
** In this case - if VEC1_USE_TRANSPOSITION is not defined, wire all
** vec1BestMatchTransposeN() calls directly to vec1BestMatchN().
*/
#define vec1BestMatchPlatformN(nElem, aCent, aTrans, nCent, aVec, aBest, p) \
vec1BestMatchN(nElem, aCent, nCent, aVec, aBest, p)
#endif
static void vec1LloydsWork(void *pCtx){
LloydsJob *p = (LloydsJob*)pCtx;
Vec1TrainVectors *pVec = p->pVec;
int nElem = pVec->nElem;
int ii;
p->fTotal = 0.0f;
memset(p->aSum, 0, sizeof_f32 * p->nK * nElem);
memset(p->aCount, 0, sizeof(int) * p->nK);
/* Loop through those vectors that are our responsibility,
** VEC1_MULTIMATCH_NVEC vectors at a time. Each iteration of this outer
** loop will compare VEC1_MULTIMATCH_NVEC training vectors to all candidate
** centroids, and update p->aCount[] and p->aSum[] with the results. */
ii = p->iFirst;
for( ; ii<=(p->iEof-VEC1_MULTIMATCH_NVEC); ii+=VEC1_MULTIMATCH_NVEC){
int vv;
const float *aVec[VEC1_MULTIMATCH_NVEC];
int aBest[VEC1_MULTIMATCH_NVEC];
/* Initialize the training vector array with the VEC1_MULTIMATCH_NVEC
** training vectors that will be considered by this loop. */
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aVec[vv] = vec1TrainingVector(pVec, ii+vv);
}
vec1BestMatchPlatformN(
nElem, p->aCentroid, p->aTrans, p->nK, aVec, aBest, &p->fTotal
);
/* Scalar array aBest[] now contains the index of the best centroid
** for each of the VEC1_MULTIMATCH_NVEC training vectors being considered
** by this loop. Update aCount[] and aSum[] avvordingly. */
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
int iBest = aBest[vv];
p->aCount[iBest]++;
vec1AddInPlace(&p->aSum[iBest*nElem], aVec[vv], nElem);
}
#if 0
/* Check (assert) using vec1BestMatchSimple() that the fancy code above
** actually worked. */
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
int iBest2 = vec1BestMatchSimple(
p->aCentroid, p->nK, aVec[vv], nElem, 0
);
if( iBest2!=aBest[vv] ){
const float *c1 = &p->aCentroid[aBest[vv] * nElem];
const float *c2 = &p->aCentroid[iBest2 * nElem];
float d1 = vec1L2Dist(aVec[vv], c1, nElem);
float d2 = vec1L2Dist(aVec[vv], c2, nElem);
assert( d1==d2
|| (d1>=d2 && ((d1 - d2) / d1) < 0.00001)
|| (d2>d1 && ((d2 - d1) / d2) < 0.00001)
);
}
}
#endif
}
/* Scalar tail - handle training vectors that were not done above as
** part of a batch of VEC1_MULTIMATCH_NVEC. */
for(; ii<p->iEof; ii++){
const float *aSub = vec1TrainingVector(pVec, ii);
int iBest = vec1BestMatchSimple(
p->aCentroid, p->nK, aSub, nElem, &p->fTotal
);
assert( iBest>=0 && iBest<p->nK );
p->aCount[iBest]++;
vec1AddInPlace(&p->aSum[iBest * nElem], aSub, nElem);
}
}
/*
** Parameter aIn points to an array of centroids containing nK vectors,
** each nElem elements in size. i.e. so that element j of vector i is
** stored in aIn[i*nElem + j]. Total size of the array is nK*nElem values.
**
** Parameter aOut[] also points to an array of nK*nElem elements. This
** function populates that array with the transposed version of the codebook
** before returning.
*/
static void vec1CodebookTranspose(
const float *aIn, /* Centroid array in on-disk format */
int nElem, /* Number of elements in each vectors */
int nWidth, /* Transpose width */
int nK, /* Number of centroids */
float *aOut /* Populate this array with transformation */
){
int K, d;
float *pOut = aOut;
/* If nK is not divisible by nWidth, just ignore any tail vectors. */
assert( (nK % nWidth)==0 || nWidth==VEC1_MULTIMATCH_NCENT );
nK = (nK / nWidth) * nWidth;
for(K=0; K<nK; K+=nWidth){
for(d=0; d<nElem; d++){
int ii;
for(ii=0; ii<nWidth; ii++){
(*pOut++) = aIn[ (K+ii)*nElem + d ];
}
}
}
assert( pOut==&aOut[ nElem * nK ] );
}
static void vec1TrainLog(Vec1TrainCtx *p, const char *zFmt, ...){
if( p->pLogStmt && p->rc==SQLITE_OK ){
int percent = ((100*p->nCompletedWork) / p->nTotalWork);
char *zMsg = 0;
va_list ap;
va_start(ap, zFmt);
zMsg = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zMsg==0 ){
p->rc = SQLITE_NOMEM;
}else{
sqlite3_bind_int(p->pLogStmt, 1, percent);
sqlite3_bind_text(p->pLogStmt, 2, zMsg, -1, SQLITE_TRANSIENT);
sqlite3_step(p->pLogStmt);
sqlite3_free(zMsg);
p->rc = sqlite3_reset(p->pLogStmt);
}
if( p->rc==SQLITE_NOMEM ){
sqlite3_result_error_nomem(p->pCtx);
}else if( p->rc!=SQLITE_OK ){
sqlite3 *db = sqlite3_db_handle(p->pLogStmt);
vec1ResultErrorF(p->pCtx, "%s", sqlite3_errmsg(db));
}
}
}
#define START_TRAINING_TIMER(p, i) { p->aTime[i] -= vec1HardwareTimer(); }
#define END_TRAINING_TIMER(p, i) { p->aTime[i] += vec1HardwareTimer(); }
static int vec1Ann1KMeans(
Vec1TrainCtx *p, /* Training data + parameters */
Vec1TrainVectors *pVec, /* Vectors to train K-Means on */
Vec1JobQueue *pQueue, /* Job-queue, or NULL for single thread */
int nK, /* Value of K in K-means */
float *aCode /* OUT: Populate this array */
){
i64 nByte = 0;
int ii;
int iIter;
int nElem = pVec->nElem;
int *pCsr = 0;
double fPrevTotal = 0.0;
int *aCount = 0;
float *aSum = 0;
LloydsJob *aJob = 0;
int rc = SQLITE_OK;
int nThread = p ? p->nThread : 1;
/* Allocate array of jobs to use for multi-threaded Lloyd's algorithm.
** We use this even if there are no worker threads - in that case
** allocate an array of 1. The algorithm parallelizes across vectors - each
** jobs is assigned a subset of the vectors to work on.
**
** Each job requires the following working space:
**
** + aCount -> (sizeof(int) * K) bytes
** + aSum -> (sizeof_f32 * nElem * K) bytes
*/
nByte = nThread * (
sizeof(LloydsJob) +
nElem * nK * sizeof_f32 + /* aSum[] */
nK * sizeof(int) /* aCount[] */
);
aJob = (LloydsJob*)sqlite3_malloc64(nByte);
if( aJob==0 ) return SQLITE_NOMEM;
memset(aJob, 0, sizeof(LloydsJob) * nThread);
pCsr = (int*)&aJob[nThread];
/* Initialize each job object in the array */
for(ii=0; ii<nThread; ii++){
LloydsJob *pJob = &aJob[ii];
/* Assign a range of the training vectors to this job. This job will
** process vectors from iFirst to (iEof-1), inclusive. */
ASSIGN_VECTORS_TO_JOB(pVec, pJob, ii, nThread);
pJob->nK = nK;
pJob->aCentroid = aCode;
pJob->aSum = (float*)pCsr;
pCsr += (nK * nElem);
pJob->aCount = pCsr;
pCsr += nK;
}
aCount = aJob[0].aCount;
aSum = aJob[0].aSum;
#ifdef VEC1_USE_TRANSPOSITION
/* If this build uses transposed arrays of centroids, allocate space
** for the array here, and give each job a pointer to it. */
aJob[0].aTrans = (float*)sqlite3_malloc64( sizeof_f32 * (nElem * nK) );
if( aJob[0].aTrans==0 ){
rc = SQLITE_NOMEM;
goto vec1_kmeans_out;
}
for(ii=1; ii<nThread; ii++){
aJob[ii].aTrans = aJob[0].aTrans;
}
#endif
assert( (u8*)pCsr==((u8*)aJob)+nByte );
/* If pQueue is not NULL, then this is training the coarse quantizer
** and so we have exclusive access to the Vec1TrainCtx object. So
** it's ok to record timings. We can't do that if pQueue is NULL,
** as in that case multiple threads may be running this code with
** the same Vec1TrainCtx object. */
if( p ){
START_TRAINING_TIMER(p, VEC1_TRAINING_COARSE_INIT);
vec1TrainLog(p, "k-means++ initalization for coarse quant");
}
rc = vec1KMeansPlusPlus(pVec, pQueue, nK, aCode);
if( rc!=SQLITE_OK ) goto vec1_kmeans_out;
if( p ){
END_TRAINING_TIMER(p, VEC1_TRAINING_COARSE_INIT);
p->nCompletedWork += VEC1_TRAINING_WORK_COARSE1;
}
/* Initial centroids have now been chosen. So do K-means iteration to
** improve the centroids. */
if( p ){
START_TRAINING_TIMER(p, VEC1_TRAINING_COARSE_KMEANS);
vec1TrainLog(p, "k-means iterations for coarse quant");
}
for(iIter=0; iIter<VEC1_PQ_KMEANS_NITER; iIter++){
double fTotal = 0.0;
int jj;
#ifdef VEC1_USE_TRANSPOSITION
vec1CodebookTranspose(
aCode, nElem, VEC1_NTRANSPOSE_WIDTH, nK, aJob[0].aTrans
);
#endif
/* Now loop through all training vectors. Extract the sub-vector associated
** with this codebook and compare it with each of the current nK centroids.
** Update the aSum[] and aCount[] entries that correspond to the closest
** centroid found. */
for(ii=0; ii<nThread; ii++){
vec1JobQueueAddJob(pQueue, vec1LloydsWork, 0, &aJob[ii]);
}
vec1JobQueueFinishJobs(pQueue, SQLITE_OK);
fTotal = aJob[0].fTotal;
for(ii=1; ii<nThread; ii++){
for(jj=0; jj<nK; jj++){
aCount[jj] += aJob[ii].aCount[jj];
}
vec1AddInPlace(aSum, aJob[ii].aSum, nElem*nK);
fTotal += aJob[ii].fTotal;
}
#if 0
/* TODO: This is in theory better, but in practice it seems to mess up
** recall when using PQ codes and residual:1. So omit this for now. */
if( p && p->eDistance==VEC1_DISTANCE_COS ){
/* If this call is training a coarse quantizer for cosine distance,
** use "spherical k-means". This means that instead of just updating
** the centroid to be the average of all points assigned to it, we
** do that and then normalize the result - so that the centroid
** remains on the surface of the unit sphere. */
memcpy(aCode, aSum, sizeof_f32 * nK * nElem);
for(ii=0; ii<nK; ii++){
vec1NormalizeVector(&aCode[ii*nElem], nElem);
}
}else
#endif
{
/* Update each centroid to be the average of points assigned to it. */
for(ii=0; ii<nK; ii++){
if( aCount[ii]>0 ){
int iElem;
for(iElem=0; iElem<nElem; iElem++){
int iCodeElem = ii*nElem + iElem;
aCode[iCodeElem] = aSum[iCodeElem] / aCount[ii];
}
}
}
}
/* If the sum of the L2-squared distortion produced by encoding the
** training set using the current codebook is greater than
** VEC1_PQ_KMEANS_STOPVAL (which might be say 0.999) times that of the
** previous iteration, end the loop now. */
if( iIter>0 && (fTotal / fPrevTotal)>=VEC1_PQ_KMEANS_STOPVAL ){
iIter = VEC1_PQ_KMEANS_NITER;
}
fPrevTotal = fTotal;
}
if( p ){
END_TRAINING_TIMER(p, VEC1_TRAINING_COARSE_KMEANS);
p->nCompletedWork += VEC1_TRAINING_WORK_COARSE2;
}
vec1_kmeans_out:
#ifdef VEC1_USE_TRANSPOSITION
sqlite3_free( aJob[0].aTrans );
#endif
sqlite3_free(aJob);
return rc;
}
/*
** This function is called to train a coarse quantizer from within a
** vec1_train() invocation. Output array aCentroid[], which must be
** (p->tv.nElem * p->nBucket) elements in size, is packed with
** the p->nBucket centroids to use for coarse quantization before
** returning.
*/
static void vec1TrainCoarseQuant(
Vec1TrainCtx *p, /* Training data + parameters */
Vec1JobQueue *pQueue, /* Job-queue, or NULL for single thread */
float *aCentroid /* OUT: Populate this array */
){
if( p->rc==SQLITE_OK ){
int rc = vec1Ann1KMeans(p, &p->tv, pQueue, p->nBucket, aCentroid);
assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){
p->rc = rc;
sqlite3_result_error_nomem(p->pCtx);
}
}
}
static int vec1ConfigInt(
int eType,
i64 iVal, /* Proposed parameter value */
int nMin, /* Minimum allowed value */
int nMax, /* Maximum allowed value */
int bAllowZero, /* True if 0 is also allowed */
const char *zName, /* Name of parameter */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK;
if( eType!=SQLITE_INTEGER
|| (!(bAllowZero && iVal==0) && (iVal<nMin || iVal>nMax))
){
*pzErr = sqlite3_mprintf(
"vec1: %s must be set to an integer value between %d and %d%s",
zName, nMin, nMax, (bAllowZero ? ", or 0" : "")
);
rc = SQLITE_ERROR;
}
return rc;
}
static int vec1ConfigBool(
int *pbOut,
int eType,
i64 iVal,
const char *zName,
char **pzErr
){
int rc = SQLITE_OK;
if( eType==SQLITE_INTEGER && (iVal==0 || iVal==1) ){
*pbOut = (iVal!=0);
}else{
*pzErr = sqlite3_mprintf("vec1: %s must be set to a boolean value", zName);
rc = SQLITE_ERROR;
}
return rc;
}
static int vec1ConfigEnum(
int *piOpt, /* OUT: Index of selected value */
const char *zVal, /* Candidate value */
const char **azOpt, /* Array of options */
const char *zName, /* Name of parameter */
char **pzErr /* OUT: Error message */
){
int ii;
char *zErr = 0;
for(ii=0; azOpt[ii]; ii++){
if( sqlite3_stricmp(zVal, azOpt[ii])==0 ){
*piOpt = ii;
return SQLITE_OK;
}
}
zErr = sqlite3_mprintf("unrecognized %s '%s', should be one of", zName, zVal);
for(ii=0; azOpt[ii]; ii++){
if( azOpt[ii+1] ){
zErr = sqlite3_mprintf("%z %s,", zErr, azOpt[ii]);
}else{
zErr = sqlite3_mprintf("%z or %s", zErr, azOpt[ii]);
}
}
*pzErr = zErr;
return SQLITE_ERROR;
}
static int vec1ConfigDistance(
int *peDist, /* OUT: VEC1_DISTANCE_L2 or DISTANCE_COS */
const char *zVal, /* Candidate value */
char **pzErr /* OUT: Error message */
){
const char *azDist[] = {"l2", "cos", 0};
int iOpt = 0;
int rc = vec1ConfigEnum(&iOpt, zVal, azDist, "distance", pzErr);
assert( VEC1_DISTANCE_L2==1 );
assert( VEC1_DISTANCE_COS==2 );
*peDist = iOpt+1;
return rc;
}
/*
** Parameter zVal points to a string that was the value for the "quantize"
** field of a training configuration object. If the string is a recognized
** value for this field, output parameter (*peQuant) is set to the
** corresponding VEC1_QUANTIZE_XXX value and SQLITE OK returned.
**
** Otherwise, if the value cannot be recognized, (*pzErr) is set to point
** to a buffer containing an English language error message and SQLITE_ERROR
** is returned. It is the responsibility of the caller to eventually free
** the (*pzErr) buffer using sqlite3_free(). The final value of (*peQuant)
** is undefined in this case.
*/
static int vec1ConfigQuantizer(
int *peQuant, /* OUT: VEC1_QUANT_XXX value */
const char *zVal, /* Candidate value */
char **pzErr /* OUT: Error message */
){
const char *azQuant[] = {"none", "pq", "opq", "bq", 0};
assert( sqlite3_stricmp("none", azQuant[VEC1_QUANTIZE_NONE])==0 );
assert( sqlite3_stricmp("pq", azQuant[VEC1_QUANTIZE_PQ])==0 );
assert( sqlite3_stricmp("opq", azQuant[VEC1_QUANTIZE_OPQ])==0 );
assert( sqlite3_stricmp("bq", azQuant[VEC1_QUANTIZE_BQ])==0 );
return vec1ConfigEnum(peQuant, zVal, azQuant, "quantizer", pzErr);
}
static int vec1Ann1TrainCfg(
void *pCtx,
const char *zOpt,
int eType,
i64 iVal,
double fVal,
const char *zVal,
char **pz
){
int rc = SQLITE_OK;
Vec1TrainCtx *p = (Vec1TrainCtx*)pCtx;
const char *azOpt[] = {
"distance", /* 0 */
"codesize", /* 1 */
"nbucket", /* 2 */
"quantizer", /* 3 */
"nthread", /* 4 */
"svd_verify", /* 5 */
"opq", /* 6 */
"progress", /* 7 */
"residual", /* 8 */
"wht", /* 9 */
"nopq_round", /* 10 */
"profile", /* 11 */
"index", /* 12 */
0
};
int eOpt = -1;
UNUSED_PARAMETER(fVal);
rc = vec1ConfigEnum(&eOpt, zOpt, azOpt, "option", pz);
if( rc==SQLITE_OK ){
switch( eOpt ){
case 0: assert( sqlite3_stricmp("distance", zOpt)==0 );
rc = vec1ConfigDistance(&p->eDistance, zVal, pz);
break;
case 1: assert( sqlite3_stricmp("codesize", zOpt)==0 );
rc = vec1ConfigInt(eType, iVal,
VEC1_MIN_CODESIZE, VEC1_MAX_CODESIZE, 1, zOpt, pz
);
p->nCodesize = (int)iVal;
break;
case 2: assert( sqlite3_stricmp("nbucket", zOpt)==0 );
rc = vec1ConfigInt(eType, iVal,
VEC1_MIN_BUCKET, VEC1_MAX_BUCKET, 1, zOpt, pz
);
p->nBucket = (int)iVal;
break;
case 3: assert( sqlite3_stricmp("quantizer", zOpt)==0 );
rc = vec1ConfigQuantizer(&p->eQuant, zVal, pz);
break;
case 4: assert( sqlite3_stricmp("nthread", zOpt)==0 );
rc = vec1ConfigInt(eType, iVal, 1, VEC1_MAX_NTHREAD, 0, zOpt, pz);
#if VEC1_THREADS
p->nThread = (int)iVal;
#endif
break;
case 5: assert( sqlite3_stricmp("svd_verify", zOpt)==0 );
rc = vec1ConfigBool(&p->bSvdVerify, eType, iVal, zOpt, pz);
break;
case 6: assert( sqlite3_stricmp("opq", zOpt)==0 );
rc = vec1ConfigBool(&p->bOpq, eType, iVal, zOpt, pz);
break;
case 7: assert( sqlite3_stricmp("progress", zOpt)==0 );
sqlite3_free(p->zLogFunction);
p->zLogFunction = sqlite3_mprintf("%s", zVal);
if( p->zLogFunction==0 ) rc = SQLITE_NOMEM;
break;
case 8: assert( sqlite3_stricmp("residual", zOpt)==0 );
rc = vec1ConfigBool(&p->bResidual, eType, iVal, zOpt, pz);
break;
case 9: assert( sqlite3_stricmp("wht", zOpt)==0 );
rc = vec1ConfigBool(&p->bWHT, eType, iVal, zOpt, pz);
break;
case 10: assert( sqlite3_stricmp("nopq_round", zOpt)==0 );
rc = vec1ConfigInt(eType, iVal,
VEC1_MIN_NOPQ_ROUND, VEC1_MAX_NOPQ_ROUND, 0, zOpt, pz
);
p->nOpqRound = (int)iVal;
break;
case 11: assert( sqlite3_stricmp("profile", zOpt)==0 );
sqlite3_free(p->zProfileFunction);
p->zProfileFunction = sqlite3_mprintf("%s", zVal);
if( p->zProfileFunction==0 ) rc = SQLITE_NOMEM;
break;
default: assert( sqlite3_stricmp("index", zOpt)==0 );
assert( eOpt==12 ); /* index */
if( sqlite3_stricmp("trained", zVal) ){
*pz = sqlite3_mprintf(
"unrecognized index '%s', should be 'trained'", zVal
);
rc = SQLITE_ERROR;
}
break;
}
}
return rc;
}
#define VEC1_INDEX_NONE 1
#define VEC1_INDEX_FLAT 2
typedef struct Vec1FlatIndex Vec1FlatIndex;
struct Vec1FlatIndex {
int eDistance;
int eIndex;
};
static int vec1FlatIndexCfg(
void *pCtx,
const char *zOpt,
int eType,
i64 iVal,
double fVal,
const char *zVal,
char **pz
){
int rc = SQLITE_OK;
Vec1FlatIndex *p = (Vec1FlatIndex*)pCtx;
UNUSED_PARAMETER(eType);
UNUSED_PARAMETER(iVal);
UNUSED_PARAMETER(fVal);
if( sqlite3_stricmp("distance", zOpt)==0 ){
rc = vec1ConfigDistance(&p->eDistance, zVal, pz);
}else
if( sqlite3_stricmp("index", zOpt)==0 ){
const char *azIdx[] = {"none", "flat", 0};
int iOpt = 0;
rc = vec1ConfigEnum(&iOpt, zVal, azIdx, "index", pz);
assert( VEC1_INDEX_NONE==1 );
assert( VEC1_INDEX_FLAT==2 );
p->eIndex = iOpt+1;
}else{
*pz = sqlite3_mprintf("unknown parameter: %s", zOpt);
rc = SQLITE_ERROR;
}
return rc;
}
/*
** aVec[] is a vector nElem elements in size. If any of these elements is
** non-finite - either NaN or an Infinity value - return SQLITE_ERROR.
** Return SQLITE_OK if all elements are finite.
*/
static int vec1VectorIsNonFinite(const float *aVec, int nElem){
static const u32 MASK = 0x7F800000;
int ii = 0;
#if defined(VEC1_HAVE_AVX2)
const __m256i exp_mask = _mm256_set1_epi32(MASK);
for(; ii<=(nElem-8); ii+=8){
__m256i bits = _mm256_loadu_si256((__m256i*)&aVec[ii]);
__m256i exp_bits = _mm256_and_si256(bits, exp_mask);
__m256i nonfinite = _mm256_cmpeq_epi32(exp_bits, exp_mask);
if( _mm256_movemask_ps(_mm256_castsi256_ps(nonfinite)) ){
goto error_out;
}
}
#endif
#if defined(VEC1_HAVE_NEON)
const uint32x4_t exp_mask = vdupq_n_u32(MASK);
for(; ii<=(nElem-4); ii+=4){
uint32x4_t bits = vld1q_u32((u32*)&aVec[ii]);
uint32x4_t exp_bits = vandq_u32(bits, exp_mask);
uint32x4_t nonfinite = vceqq_u32(exp_bits, exp_mask);
if( vmaxvq_u32(nonfinite) ){
goto error_out;
}
}
#endif
for(; ii<nElem; ii++){
if( (((u32*)aVec)[ii] & MASK)==MASK ){
goto error_out;
}
}
return SQLITE_OK;
error_out:
return SQLITE_ERROR;
}
#define VEC1_REALLOC_NVEC 10000
/*
** The xStep() method for the vec1_train() aggregate function:
**
** vec1_train(VECTOR, CONFIG)
**
** All that this step function does is populate the Vec1TrainCtx.aVec[] array.
** The interesting stuff happens in vec1TrainFinal().
*/
static void vec1TrainStep(
sqlite3_context *pCtx,
int nArg,
sqlite3_value **aArg
){
Vec1TabList *pList = (Vec1TabList*)sqlite3_user_data(pCtx);
Vec1TrainCtx *p = 0;
assert( nArg==1 || nArg==2 );
int n = 0;
const u8 *a = 0;
int iChunk = 0;
p = (Vec1TrainCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p));
if( !p ) return;
n = sqlite3_value_bytes(aArg[0]);
a = (const u8*)sqlite3_value_blob(aArg[0]);
if( (n==0)
|| (n%sizeof_f32)!=0
|| (p->tv.nElem!=0 && p->tv.nElem!=(int)(n/sizeof_f32))
){
sqlite3_result_error(pCtx, "vec1_train: bad argument", -1);
return;
}
if( vec1VectorIsNonFinite((const float*)a, (n/sizeof_f32)) ){
sqlite3_result_error(pCtx,
"vec1_train: non-finite value not allowed in vector", -1
);
return;
}
/* If this is the first call to xStep() for this aggregate, initialize
** the various "training parameters" fields of Vec1TrainCtx. */
if( p->tv.nElem==0 ){
p->tv.nElem = n/sizeof_f32;
p->tv.nVecPerChunk = (VEC1_TRAINING_SZCHUNK / (p->tv.nElem*sizeof_f32));
/* Configure default values */
p->eDistance = VEC1_TRAINING_DEFAULT_DISTANCE;
p->nBucket = VEC1_TRAINING_DEFAULT_NBUCKET;
p->nThread = pList->nThread;
p->nOpqRound = VEC1_TRAINING_DEFAULT_NOPQ_ROUND;
p->bSvdVerify = VEC1_TRAINING_DEFAULT_SVD_VERIFY;
p->bResidual = VEC1_TRAINING_DEFAULT_RESIDUAL;
p->eQuant = -1;
p->nCodesize = -1;
p->bWHT = -1;
p->bOpq = -1;
/* If a configuration object was passed, update the Vec1TrainCtx with
** its field values now. */
if( nArg==2 ){
sqlite3 *db = sqlite3_context_db_handle(pCtx);
const char *zCfg = (const char*)sqlite3_value_text(aArg[1]);
char *zErr = 0;
int rc = vec1ParseJsonConfig(db, zCfg, vec1Ann1TrainCfg, (void*)p, &zErr);
if( rc ){
if( zErr!=0 ){
vec1ResultErrorF(pCtx, "%z", zErr);
}else{
sqlite3_result_error_code(pCtx, rc);
}
return;
}
}
/* If no quantizer was specified, select one automatically now. */
if( p->eQuant<0 ){
if( p->nCodesize<=0 ){
p->eQuant = VEC1_QUANTIZE_NONE;
}else if( p->bOpq>0 ){
p->eQuant = VEC1_QUANTIZE_OPQ;
}else{
p->eQuant = VEC1_QUANTIZE_PQ;
}
}
/* If no "wht" value was explicitly specified, enable the Walsh-Hadamard
** Transform for binary quantizers. Disable it for other quantizers. */
if( p->bWHT<0 ) p->bWHT = (p->eQuant==VEC1_QUANTIZE_BQ);
/* If no value was specified for "codesize", choose a default. This
** is 32 for PQ or OPQ, 0 for "none" quantizer, and (nElem+7)/8 for BQ. */
if( p->eQuant==VEC1_QUANTIZE_NONE ){
p->nCodesize = 0;
p->bResidual = 0;
}else if( p->eQuant==VEC1_QUANTIZE_BQ ){
if( p->nCodesize<=0 || p->nCodesize>=((p->tv.nElem+7)/8) ){
/* No explicit codesize. Or an explicit codesize large enough to
** use 1 bit per dimension. So do that - use 1 bit per dimension. */
p->nCodesize = (p->tv.nElem + 7) / 8;
}else{
/* Explicit codesize too small for 1 bit per dimension. So use 1 bit
** for each two dimensions. */
p->nCodesize = (p->tv.nElem + 15) / 16;
}
}else if( p->nCodesize<=0 ){
if( p->eQuant==VEC1_QUANTIZE_PQ ){
p->nCodesize = (p->tv.nElem + 7) / 8;
}else{
assert( p->eQuant==VEC1_QUANTIZE_OPQ );
p->nCodesize = 32;
}
}
if( p->eQuant==VEC1_QUANTIZE_PQ || p->eQuant==VEC1_QUANTIZE_OPQ ){
/* Usually, tv.nElem will be an integer multiple of nCodebook. In
** this case set nCodeElem is just set to (tv.nElem / nCodebook) and
** nCodebook remains as configured.
**
** Or, if tv.nElem is not a multiple of nCodebook, set nCodeElem to
** the smallest value that allows us to cover the entire vector with
** nCodebook codebooks. Then reduce nCodebook to the minimum number
** of codebooks required to cover the vector with nCodeElem
** sub-vectors. */
p->nCodeElem = ((p->tv.nElem+p->nCodesize-1) / p->nCodesize);
p->nCodesize = ((p->tv.nElem+p->nCodeElem-1) / p->nCodeElem);
}
}
iChunk = (p->tv.nVec+1)/p->tv.nVecPerChunk;
if( iChunk>=p->tv.nChunk ){
p->tv.aChunk[iChunk] = (float*)sqlite3_malloc(VEC1_TRAINING_SZCHUNK);
if( p->tv.aChunk[iChunk]==0 ){
sqlite3_result_error_nomem(pCtx);
return;
}
p->tv.nChunk++;
}
/* Append a copy of the vector to Vec1TrainCtx.aVec[]. */
memcpy(vec1TrainingVector(&p->tv, p->tv.nVec), sqlite3_value_blob(aArg[0]),n);
p->tv.nVec++;
}
/*
** Normalize all the training vectors.
*/
static void vec1TrainNormalizeAll(Vec1TrainCtx *p){
int ii;
for(ii=0; ii<p->tv.nVec; ii++){
vec1NormalizeVector(vec1TrainingVector(&p->tv, ii), p->tv.nElem);
}
}
static void vec1FreeVectors(Vec1TrainVectors *pFree){
if( pFree ){
int ii;
for(ii=0; ii<pFree->nChunk; ii++){
sqlite3_free(pFree->aChunk[ii]);
}
sqlite3_free(pFree);
}
}
/*
** An instance of the following structure is allocated for each PQ codebook
** to train. Each codebook can then be trained in a separate thread.
*/
typedef struct Vec1CodebookJob Vec1CodebookJob;
struct Vec1CodebookJob {
Vec1TrainCtx *p;
Vec1TrainVectors *pTrain;
int iOff;
float *aCode;
int rc;
};
static void vec1CodebookWork(void *pCtx){
Vec1CodebookJob *pJob = (Vec1CodebookJob*)pCtx;
Vec1TrainCtx *p = pJob->p;
int nByte = sizeof(Vec1TrainVectors);
Vec1TrainVectors *pVec = (Vec1TrainVectors*)vec1MallocZero(nByte);
if( pVec==0 ){
pJob->rc = SQLITE_NOMEM;
}else{
int ii;
int nShort = 0;
int nBytePerVec = p->nCodeElem * sizeof_f32;
pVec->nElem = p->nCodeElem;
pVec->nVec = pJob->pTrain->nVec;
pVec->nVecPerChunk = (VEC1_TRAINING_SZCHUNK / nBytePerVec);
pVec->nChunk = (pVec->nVec+pVec->nVecPerChunk-1) / pVec->nVecPerChunk;
/* If the training vector size is not an integer multiple of the
** sub-vector size and this is the rightmost sub-vector (the one that
** will need to be padded with 0.0 values), set nShort to non-zero. */
if( (pJob->iOff + p->nCodeElem)>pJob->pTrain->nElem ){
nShort = pJob->pTrain->nElem - pJob->iOff;
}
/* Allocate chunks for the new Vec1TrainVectors structure. If nShort
** is set, also zero them out. */
for(ii=0; ii<pVec->nChunk; ii++){
/* Allocate space for this chunk. Either VEC1_TRAINING_SZCHUNK or
** enough space for all remaining sub-vectors, whichever is smaller. */
int nAlloc = MIN(VEC1_TRAINING_SZCHUNK,
(pVec->nVec - (ii * pVec->nVecPerChunk)) * nBytePerVec
);
pVec->aChunk[ii] = (float*)sqlite3_malloc(nAlloc);
if( pVec->aChunk[ii]==0 ){
pJob->rc = SQLITE_NOMEM;
break;
}
if( nShort>0 ){
memset(pVec->aChunk[ii], 0, nAlloc);
}
}
if( pJob->rc==SQLITE_OK ){
int nCopy = (nShort ? nShort : pVec->nElem) * sizeof_f32;
for(ii=0; ii<pVec->nVec; ii++){
float *aVec = vec1TrainingVector(pJob->pTrain, ii);
memcpy(vec1TrainingVector(pVec, ii), &aVec[pJob->iOff], nCopy);
}
pJob->rc = vec1Ann1KMeans(0, pVec, 0, VEC1_PQ_CODEBOOK_SZ, pJob->aCode);
}
vec1FreeVectors(pVec);
}
}
static int vec1CodebookFinal(void *pCtx, int rcin){
UNUSED_PARAMETER(rcin);
return ((Vec1CodebookJob*)pCtx)->rc;
}
static void vec1TrainPQCodebooks(
Vec1TrainCtx *p,
Vec1TrainVectors *pVec,
Vec1JobQueue *pQueue,
float *aBook
){
if( p->rc==SQLITE_OK ){
Vec1CodebookJob *aJob = 0;
aJob = vec1MallocZero(sizeof(Vec1CodebookJob) * p->nCodesize);
if( aJob ){
int ii;
int rc = SQLITE_OK;
for(ii=0; rc==SQLITE_OK && ii<p->nCodesize; ii++){
Vec1CodebookJob *pJob = &aJob[ii];
pJob->p = p;
pJob->iOff = ii * p->nCodeElem;
pJob->aCode = &aBook[ii * p->nCodeElem * VEC1_PQ_CODEBOOK_SZ];
pJob->rc = SQLITE_OK;
pJob->pTrain = pVec;
rc = vec1JobQueueAddJob(
pQueue, vec1CodebookWork, vec1CodebookFinal, pJob
);
}
p->rc = vec1JobQueueFinishJobs(pQueue, rc);
}else{
p->rc = SQLITE_NOMEM;
}
if( p->rc!=SQLITE_OK ){
sqlite3_result_error_code(p->pCtx, p->rc);
}
sqlite3_free(aJob);
}
}
static void vec1TransposeMatrix(double *aMatrix, int nElem) {
for (int i = 0; i < nElem - 1; i++) {
for (int j = i + 1; j < nElem; j++) {
double tmp = aMatrix[i * nElem + j];
aMatrix[i * nElem + j] = aMatrix[j * nElem + i];
aMatrix[j * nElem + i] = tmp;
}
}
}
static int vec1JacobiDoOnePair(
int p, int q,
double *A,
int d,
double *Vt,
double fTol
){
double alpha = 0.0, beta = 0.0, gamma = 0.0;
int r;
double *Ap = A + p*d;
double *Aq = A + q*d;
assert( p<q );
r = 0;
#ifdef VEC1_HAVE_AVX2
__m256d valpha = _mm256_setzero_pd();
__m256d vbeta = _mm256_setzero_pd();
__m256d vgamma = _mm256_setzero_pd();
for(/*no-op*/; r<(d-3); r+=4) {
__m256d vAp = _mm256_loadu_pd(Ap + r);
__m256d vAq = _mm256_loadu_pd(Aq + r);
valpha = FMADD64(vAp, vAp, valpha);
vbeta = FMADD64(vAq, vAq, vbeta);
vgamma = FMADD64(vAp, vAq, vgamma);
}
HORIZONTAL_SUM64(alpha, valpha);
HORIZONTAL_SUM64(beta, vbeta);
HORIZONTAL_SUM64(gamma, vgamma);
#endif
#ifdef VEC1_HAVE_NEON
float64x2_t valpha = vdupq_n_f64(0.0);
float64x2_t vbeta = vdupq_n_f64(0.0);
float64x2_t vgamma = vdupq_n_f64(0.0);
for(/*no-op*/; r<=(d-2); r+=2) {
float64x2_t vAp = vld1q_f64(&Ap[r]);
float64x2_t vAq = vld1q_f64(&Aq[r]);
valpha = vfmaq_f64(valpha, vAp, vAp);
vbeta = vfmaq_f64(vbeta, vAq, vAq);
vgamma = vfmaq_f64(vgamma, vAp, vAq);
}
alpha = vaddvq_f64(valpha);
beta = vaddvq_f64(vbeta);
gamma = vaddvq_f64(vgamma);
#endif
for(/*no-op*/; r<d; r++) {
alpha += Ap[r] * Ap[r];
beta += Aq[r] * Aq[r];
gamma += Ap[r] * Aq[r];
}
if( gamma!=0.0 && fabs(gamma)>=fTol*sqrt(alpha*beta) ){
double zeta = (beta - alpha) / (2.0 * gamma);
double t, c, s;
if (zeta >= 0.0){
t = 1.0 / ( zeta + sqrt(1.0 + zeta * zeta));
}else{
t = -1.0 / (-zeta + sqrt(1.0 + zeta * zeta));
}
/* Damping factor apparently sometimes required by parallel Jacobi.
** Wait to see if we have a problem first I suppose... */
/* t = t*0.5; */
c = 1.0 / sqrt(1.0 + t * t);
s = -c * t;
/* Rotate columns p and q of A */
r = 0;
#ifdef VEC1_HAVE_AVX2
__m256d vc = _mm256_set1_pd(c);
__m256d vs = _mm256_set1_pd(s);
__m256d vms = _mm256_set1_pd(-s);
for(; r<d-3; r+=4) {
__m256d vap = _mm256_loadu_pd(&Ap[r]);
__m256d vaq = _mm256_loadu_pd(&Aq[r]);
_mm256_storeu_pd(&Ap[r], FMADD64(vc, vap, _mm256_mul_pd(vs, vaq)));
_mm256_storeu_pd(&Aq[r], FMADD64(vms, vap, _mm256_mul_pd(vc, vaq)));
}
#endif
#ifdef VEC1_HAVE_NEON
float64x2_t vc = vdupq_n_f64(c);
float64x2_t vs = vdupq_n_f64(s);
float64x2_t vms = vdupq_n_f64(-s);
for(; r<=d-2; r+=2) {
float64x2_t vap = vld1q_f64(&Ap[r]);
float64x2_t vaq = vld1q_f64(&Aq[r]);
vst1q_f64(&Ap[r], vfmaq_f64(vmulq_f64(vs, vaq), vc, vap));
vst1q_f64(&Aq[r], vfmaq_f64(vmulq_f64(vc, vaq), vms, vap));
}
#endif
for(/*no-op*/; r<d; r++) {
double wp = Ap[r];
double wq = Aq[r];
Ap[r] = c * wp + s * wq;
Aq[r] = -s * wp + c * wq;
}
/* Accumulate into Vt. Vt is accumulated here in ROW-MAJOR
** format. This is so that rows are contiguous to avoid thrashing
** the cache.
**
** We are building V by right-multiplying by each Jacobi
** rotation G(p,q,theta). Equivalently, we update Vt by
** left-multiplying its rows p and q.
**
** Vt(p, r) = c * Vt(p,r) + s * Vt(q,r)
** Vt(q, r) = -s * Vt(p,r_old) + c * Vt(q,r)
*/
double *Vtp = Vt + p * d;
double *Vtq = Vt + q * d;
r = 0;
#ifdef VEC1_HAVE_AVX2
for(; r<d-3; r+=4) {
__m256d vvp = _mm256_loadu_pd(&Vtp[r]);
__m256d vvq = _mm256_loadu_pd(&Vtq[r]);
_mm256_storeu_pd(&Vtp[r], FMADD64(vc, vvp, _mm256_mul_pd(vs, vvq)));
_mm256_storeu_pd(&Vtq[r], FMADD64(vms,vvp, _mm256_mul_pd(vc, vvq)));
}
#endif
#ifdef VEC1_HAVE_NEON
for (; r<=d-2; r+=2) {
float64x2_t vvp = vld1q_f64(&Vtp[r]);
float64x2_t vvq = vld1q_f64(&Vtq[r]);
vst1q_f64(&Vtp[r], vfmaq_f64(vmulq_f64(vs, vvq), vc, vvp));
vst1q_f64(&Vtq[r], vfmaq_f64(vmulq_f64(vc, vvq), vms, vvp));
}
#endif
for(; r < d; r++){
double vp = Vtp[r];
double vq = Vtq[r];
Vtp[r] = c * vp + s * vq;
Vtq[r] = -s * vp + c * vq;
}
return 1;
}
return 0;
}
/*
** Return the kth pair of round r of a round-robin with n contestents.
*/
static void vec1RoundRobinPair(
int n, /* Number of elements to pair */
int r, /* Round number [0..n-2] */
int k, /* Pair number [0..(n/2)-1] */
int *i, /* First element of pair */
int *j /* Second element of pair */
) {
assert( (n%2)==0 );
if( k==0 ){
*i = 0;
*j = (n - 1 - r);
} else {
int a = ((n-1)+k - 1 - r) % (n-1);
int b = ((n-2)+(n-k-1) - r) % (n-1);
*i = a + 1;
*j = b + 1;
}
}
#if 0
static int vec1TestRoundRobin(sqlite3_context *pCtx){
#define VEC1_RR_SZTEST 512
int aCol[VEC1_RR_SZTEST];
int aSeen[VEC1_RR_SZTEST];
int r;
int k;
for(k=0; k<VEC1_RR_SZTEST; k++) aCol[k] = k;
for(r=0; r<(VEC1_RR_SZTEST-1); r++){
int last = aCol[VEC1_RR_SZTEST-1];
memset(aSeen, 0, sizeof(int)*VEC1_RR_SZTEST);
for(k=0; k<(VEC1_RR_SZTEST/2); k++){
int a, b;
vec1RoundRobinPair(VEC1_RR_SZTEST, r, k, &a, &b);
if( a!=aCol[k] || b!=aCol[VEC1_RR_SZTEST-k-1] ){
vec1ResultErrorF(pCtx, "error in round-robin test");
return SQLITE_ERROR;
}
if( aSeen[a] || aSeen[b] ){
vec1ResultErrorF(pCtx, "big error in round-robin test");
}
aSeen[a] = 1;
aSeen[b] = 1;
}
/* Hold aCol[0] constant, rotate the remaining elements of the vector. */
memmove(&aCol[2], &aCol[1], sizeof(int)*(VEC1_RR_SZTEST-2));
aCol[1] = last;
}
return SQLITE_OK;
}
#endif
/*
** Each job in the paralell blocked jacobi transformation is represented
** by an instance of this structure.
*/
typedef struct Vec1JacobiJob Vec1JacobiJob;
struct Vec1JacobiJob {
double *A; /* Input matrix (column-major) */
int nElem; /* Matrices are nElem*nElem */
double *Vt; /* Vt matrix (row-major) */
double fTol; /* Tolerance constant */
int iJob; /* Job number */
int r; /* Current round-robin round */
int nColPerBlk; /* Number of columns in each block */
int nBlk; /* Number of blocks */
int nApply; /* Number of rotations applied this sweep. */
};
static Vec1JacobiJob *vec1JacobiAlloc(
Vec1TrainCtx *p,
double *A,
int nElem,
double *Vt,
double fTol,
int *pnJob, /* OUT: Number of objects in returned array */
int *pnStep
){
int nJob = p->nThread + (p->nThread==1);
int nBlk = nJob*2;
int nByte = nJob * sizeof(Vec1JacobiJob);
Vec1JacobiJob *aJob = (Vec1JacobiJob*)vec1MallocZero(nByte);
*pnJob = 0;
if( aJob==0 ){
p->rc = SQLITE_NOMEM;
sqlite3_result_error_nomem(p->pCtx);
}else{
int ii;
int nColPerBlk = (nElem + nBlk-1)/ nBlk;
for(ii=0; ii<nJob; ii++){
Vec1JacobiJob *pJob = &aJob[ii];
pJob->A = A;
pJob->nElem = nElem;
pJob->Vt = Vt;
pJob->fTol = fTol;
pJob->iJob = ii;
pJob->nColPerBlk = nColPerBlk;
pJob->nBlk = nBlk;
}
*pnJob = nJob;
*pnStep = nBlk-1;
}
return aJob;
}
static void vec1JacobiWork(void *pArg){
Vec1JacobiJob *pJob = (Vec1JacobiJob*)pArg;
int iBlk1 = 0;
int iBlk2 = 0;
int p0, p1, q0, q1;
int p;
vec1RoundRobinPair(pJob->nBlk, pJob->r, pJob->iJob, &iBlk1, &iBlk2);
if( iBlk1>iBlk2 ){
SWAP(int, iBlk1, iBlk2);
}
p0 = iBlk1 * pJob->nColPerBlk;
q0 = iBlk2 * pJob->nColPerBlk;
p1 = MIN(p0 + pJob->nColPerBlk, pJob->nElem);
q1 = MIN(q0 + pJob->nColPerBlk, pJob->nElem);
/* Do cross-block pairs */
for(p=p0; p<p1; p++){
int q;
for(q=q0; q<q1; q++){
pJob->nApply += vec1JacobiDoOnePair(
p, q, pJob->A, pJob->nElem, pJob->Vt, pJob->fTol
);
}
}
/* If this is round 0, also do intra-block pairs */
if( pJob->r==0 ){
for(p=p0; p<p1; p++){
int q;
for(q=p+1; q<p1; q++){
pJob->nApply += vec1JacobiDoOnePair(
p, q, pJob->A, pJob->nElem, pJob->Vt, pJob->fTol
);
}
}
for(p=q0; p<q1; p++){
int q;
for(q=p+1; q<q1; q++){
pJob->nApply += vec1JacobiDoOnePair(
p, q, pJob->A, pJob->nElem, pJob->Vt, pJob->fTol
);
}
}
}
}
/*
** This routine computes SVD of a column-major square matrix A (d x d):
**
** A = U * diag(S) * Vt
**
** Parameters:
** A - input [d*d], column-major. MODIFIED BY THIS ROUTINE
** d - matrix dimension
** U - output [d*d], column-major. Columns = left singular vectors.
** S - output [d]. Singular values, not sorted.
** Vt - output [d*d], column-major storage of V^T.
** Row j of V^T (right singular vector j) occupies
** column j in memory: elements Vt[j*d .. j*d+d-1].
** nMaxIter - max sweeps
** fTol - convergence tolerance
**
** Returns number of sweeps performed.
**/
static void vec1JacobiSVD(
Vec1TrainCtx *pTrain,
Vec1JobQueue *pQueue,
double *A,
int d,
double *U,
double *S,
double *Vt,
int nMaxIter, /* Maximum number of sweeps */
double fTol
){
int j = 0; /* Iterator */
int nSweep = 0; /* Number of sweeps performed so far */
int nStep = 0; /* Number of steps per sweep */
Vec1JacobiJob *aJob = 0;
int nJob = 0;
assert( nMaxIter>0 && fTol>0.0 );
aJob = vec1JacobiAlloc(pTrain, A, d, Vt, fTol, &nJob, &nStep);
if( aJob==0 ) return;
/* Vt is initialised to an identity matrix. */
memset(Vt, 0, d * d * sizeof(double));
for(j=0; j<d; j++){
Vt[j*d + j] = 1.0;
}
for(nSweep=0; nSweep<nMaxIter; nSweep++){
int iStep;
for(j=0; j<nJob; j++){
aJob[j].nApply = 0;
}
for(iStep=0; iStep<nStep; iStep++){
for(j=0; j<nJob; j++){
aJob[j].r = iStep;
vec1JobQueueAddJob(pQueue, vec1JacobiWork, 0, &aJob[j]);
}
vec1JobQueueFinishJobs(pQueue, SQLITE_OK);
}
for(j=0; j<nJob; j++){
if( aJob[j].nApply>0 ) break;
}
if( j==nJob ){
nSweep = nMaxIter;
}
}
/* Vt was accumulated in row-major format. Flip it back to column-major */
vec1TransposeMatrix(Vt, d);
/*
** Columns of A are now orthogonal. The singular value j is the norm
** of column j of A. Column j of U is column j of A, normalised.
*/
for(j=0; j<d; j++){
int r;
double *Aj = A + j * d;
double *Uj = U + j * d;
double norm = 0.0;
for(r=0; r<d; r++){
norm += Aj[r] * Aj[r];
}
norm = sqrt(norm);
S[j] = norm;
if( norm>fTol ){
for (r=0; r<d; r++) Uj[r] = Aj[r] / norm;
} else {
for (r=0; r<d; r++) Uj[r] = (r == j) ? 1.0 : 0.0;
}
}
sqlite3_free(aJob);
}
/*
** Compute:
**
** aOut = a1 x a2
**
** All matrixes are in column-major (Fortran) order.
*/
static void vec1MatrixMultiply(
float *aOut, /* (nElem*nElem) output array */
double *a1,
double *a2,
int nElem
){
int j;
memset(aOut, 0, sizeof_f32*nElem*nElem);
for(j=0; j<nElem; j++){ /* column of aOut */
int k;
for(k=0; k<nElem; k++){ /* column of a1 / row of a2 */
double a2_kj = a2[k + j*nElem];
int i;
for (i = 0; i < nElem; i++) { /* row of aOut */
aOut[i + j*nElem] += (float)(a1[i + k*nElem] * a2_kj);
}
}
}
}
/*
** Compute:
**
** aOut = a1 x a2
**
** All matrixes are in column-major (Fortran) order. The difference between
** this routine and vec1MatrixMultiply() is that the input matrices are
** composed of type float for this function.
*/
static void vec1MatrixMultiplyF(
float *aOut, /* (nElem*nElem) output array */
float *a1, /* (nElem*nElem) input matrix 1 */
float *a2, /* (nElem*nElem) input matrix 2 */
int nElem /* Size of matrices */
){
int j;
memset(aOut, 0, sizeof_f32*nElem*nElem);
for(j=0; j<nElem; j++){ /* column of aOut */
int k;
for(k=0; k<nElem; k++){ /* column of a1 / row of a2 */
float a2_kj = a2[k + j*nElem];
int i;
for (i = 0; i < nElem; i++) { /* row of aOut */
aOut[i + j*nElem] += a1[i + k*nElem] * a2_kj;
}
}
}
}
/*
** aMatrix is an nElem*nElem matrix in column-major format. It is supposed to
** be orthogonal, implying that:
**
** * The dot-product of each pair of columns is 0.0, and
** * The magnitude of each column is 1.0.
**
** This function tests these properties and returns the sum of the errors.
** If the returned value is very close to 0.0 then the matrix is orthogonal.
** This is used as part of internal verification of the SVD calculation.
*/
static double vec1TestOrthogonality(const double *aMatrix, int nElem) {
double err = 0.0;
for (int i = 0; i < nElem; i++) {
for (int j = 0; j < nElem; j++) {
/* dot product of column i and column j */
double dot = 0.0;
int r;
for(r=0; r<nElem; r++){
dot += aMatrix[i*nElem + r] * aMatrix[j*nElem + r];
}
/* subtract identity */
double diff = dot - (i == j ? 1.0 : 0.0);
err += diff * diff;
}
}
return sqrt(err);
}
/*
** Parameter A is an nElem*nElem matrix in column major format that has
** just undergone SVD, the outputs of which were U (nElem*nElem matrix),
** S (the nElem singular values from diagonal matrix E) and VT
** (another nElem*nElem matrix.
**
** This function tests that the product of U.E.VT is in fact A, and
** returns the magnitude of the discrepancy as a real value. If the
** returned value is close to 0.0, then the U.E.VT is a good decomposition
** of A.
*/
static double vec1TestReconstruction(
const double *A, /* input matrix [nElem*nElem], column-major */
const double *U, /* left singular vectors [nElem*nElem], column-major */
const double *S, /* singular values [nElem] */
const double *Vt, /* right singular vectors [nElem*nElem], column-major */
int nElem
) {
double err = 0.0;
double nrm = 0.0;
int row, col;
for(row=0; row<nElem; row++){
for(col=0; col<nElem; col++){
/* Compute (U * diag(S) * Vt)[row, col] */
double val = 0.0;
double a, diff;
for (int k = 0; k < nElem; k++){
val += U[k*nElem + row] * S[k] * Vt[col*nElem + k];
}
a = A[col*nElem + row];
diff = a - val;
err += diff * diff;
nrm += a * a;
}
}
return sqrt(err/nrm);
}
/*
** Check that U, S and Vt represent an SVD of matrix A. Return SQLITE_OK
** if they do, or an SQLite error code otherwise.
**
** U, S and Vt are considered a valid SVD of A if:
**
** + U is an orthogonal matrix, and
** + Vt is an orthogonal matrix, and
** + U.S.Vt is equal to A.
*/
static int vec1SVDSelfTest(
sqlite3_context *pCtx,
const double *A, /* input matrix, column-major */
const double *U, /* left singular vector matrix, column-major */
const double *S, /* singular values */
const double *Vt, /* right singular vector matrix , column-major */
int nElem
){
const double threshold = 0.00001;
const double fOrthU = vec1TestOrthogonality(U, nElem);
const double fOrthVT = vec1TestOrthogonality(Vt, nElem);
const double fReconstruction = vec1TestReconstruction(A, U, S, Vt, nElem);
/* COVERAGE: These three branches are difficult to hit, as the SVD
** computation always works. */
if( fOrthU>threshold || fOrthVT>threshold || fReconstruction>threshold ){
vec1ResultErrorF(pCtx, "vec1: SVD error - "
"fOrthU=%f, fOrthVT=%f, fReconstruction=%f",
fOrthU, fOrthVT, fReconstruction
);
return SQLITE_ERROR;
}
return SQLITE_OK;
}
typedef struct Vec1CovarianceJob Vec1CovarianceJob;
struct Vec1CovarianceJob {
double *aM; /* output matrix - each job writes distinct rows */
float *aBook; /* codebook */
Vec1TrainVectors *pVec;
int nCodeElem; /* Number of elements in each subspace */
int iSub; /* Our job to do this subspace */
double fTotalDist; /* private accumulator */
#ifdef VEC1_USE_TRANSPOSITION
float *aTrans; /* Transposed version of this job's codebook */
#endif
};
/*
** One job per sub-space.
*/
static Vec1CovarianceJob *vec1CovarianceAlloc(
Vec1TrainCtx *p,
double *aM,
Vec1TrainVectors *pVec,
float *aBook,
int *pnJob /* OUT: number of jobs */
){
int nByte = sizeof(Vec1CovarianceJob) * p->nCodesize;
Vec1CovarianceJob *aJob;
#ifdef VEC1_USE_TRANSPOSITION
/* Add space for each job to create a transposed version of its codebook */
nByte += sizeof_f32 * p->nCodeElem * VEC1_PQ_CODEBOOK_SZ * p->nCodesize;
#endif
aJob = (Vec1CovarianceJob*)vec1MallocZero(nByte);
assert( p->rc==SQLITE_OK );
*pnJob = 0;
if( aJob ){
int ii;
#ifdef VEC1_USE_TRANSPOSITION
float *aTrans = (float*)&aJob[p->nCodesize];
for(ii=0; ii<p->nCodesize; ii++){
aJob[ii].aTrans = &aTrans[ii * p->nCodeElem * VEC1_PQ_CODEBOOK_SZ];
}
#endif
for(ii=0; ii<p->nCodesize; ii++){
Vec1CovarianceJob *pJob = &aJob[ii];
pJob->aM = aM;
pJob->aBook = aBook;
pJob->pVec = pVec;
pJob->nCodeElem = p->nCodeElem;
pJob->iSub = ii;
}
*pnJob = p->nCodesize;
}else{
sqlite3_result_error_nomem(p->pCtx);
p->rc = SQLITE_NOMEM;
}
return aJob;
}
#if defined(VEC1_HAVE_AVX2)
/*
** Convert each element of the 8-float vector fVec to a double, and add
** each scalar value to the corresponding aOut[] element.
*/
static void vec1AddToDoubleArrayAVX2(double *aOut, __m256 fVec){
__m256d lo = _mm256_cvtps_pd(_mm256_castps256_ps128(fVec) );
__m256d hi = _mm256_cvtps_pd(_mm256_extractf128_ps(fVec, 1));
__m256d lo1 = LOADU64(&aOut[0]);
__m256d hi1 = LOADU64(&aOut[4]);
_mm256_storeu_pd(&aOut[0], _mm256_add_pd(lo, lo1));
_mm256_storeu_pd(&aOut[4], _mm256_add_pd(hi, hi1));
}
#endif
#if defined(VEC1_HAVE_NEON)
/*
** Convert each element of the 4-float vector fVec to a double, and add
** each scalar value to the corresponding aOut[] element.
*/
static void vec1AddToDoubleArrayNEON(double *aOut, float32x4_t fVec){
float64x2_t lo = vcvt_f64_f32(vget_low_f32(fVec));
float64x2_t hi = vcvt_high_f64_f32(fVec);
float64x2_t lo1 = vld1q_f64(&aOut[0]);
float64x2_t hi1 = vld1q_f64(&aOut[2]);
vst1q_f64(&aOut[0], vaddq_f64(lo, lo1));
vst1q_f64(&aOut[2], vaddq_f64(hi, hi1));
}
#endif
/*
** The number of vectors handled by each pass in vec1CovarianceWork().
*/
#define VEC1_COVAR_NVEC (VEC1_MULTIMATCH_NVEC * VEC1_COVAR_MULTIMATCH_MULT)
/*
** For each vv in (0..VEC1_COVAR_NVEC-1), and each ii in
** (0..nElem-1), do the following:
**
** aOut[ii] = aOut[ii] + SUM(aScalar[vv] * aIn[vv][ii])
*/
static void vec1CovarianceAccumulateN(
double *aOut,
const float *aIn[VEC1_COVAR_NVEC],
const float aScalar[VEC1_COVAR_NVEC],
int nElem
){
int ii = 0;
int vv;
#ifdef VEC1_HAVE_AVX2
#define VEC1_COVAR_NACC 8
__m256 vs[VEC1_COVAR_NVEC];
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
vs[vv] = _mm256_set1_ps(aScalar[vv]);
}
for(; ii<=(nElem-(8*VEC1_COVAR_NACC)); ii+=(8*VEC1_COVAR_NACC)){
__m256 aAcc[VEC1_COVAR_NACC];
int aa;
for(aa=0; aa<VEC1_COVAR_NACC; aa++){
aAcc[aa] = _mm256_set1_ps(0.0f);
}
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
for(aa=0; aa<VEC1_COVAR_NACC; aa++){
__m256 vf = LOADU(&aIn[vv][ii+8*aa]);
aAcc[aa] = FMADD(vs[vv], vf, aAcc[aa]);
}
}
for(aa=0; aa<VEC1_COVAR_NACC; aa++){
vec1AddToDoubleArrayAVX2(&aOut[ii+8*aa], aAcc[aa]);
}
}
for(; ii<=(nElem-8); ii+=8){
__m256 acc = _mm256_set1_ps(0.0f);
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
__m256 vf = LOADU(&aIn[vv][ii]);
acc = FMADD(vs[vv], vf, acc);
}
vec1AddToDoubleArrayAVX2(&aOut[ii], acc);
}
#endif /* VEC1_HAVE_AVX2 */
#ifdef VEC1_HAVE_NEON
/* Chosen so that VEC1_COVAR_NACC_NEON + VEC1_COVAR_NVEC (the vs[] array)
** stays comfortably within the 24 caller-saved NEON vector registers,
** to avoid spilling vs[] to the stack inside the hot loop. */
#define VEC1_COVAR_NACC_NEON (16/VEC1_COVAR_MULTIMATCH_MULT)
float32x4_t vs[VEC1_COVAR_NVEC];
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
vs[vv] = vdupq_n_f32(aScalar[vv]);
}
for(; ii<=(nElem-(4*VEC1_COVAR_NACC_NEON)); ii+=(4*VEC1_COVAR_NACC_NEON)){
float32x4_t aAcc[VEC1_COVAR_NACC_NEON];
int aa;
for(aa=0; aa<VEC1_COVAR_NACC_NEON; aa++){
aAcc[aa] = vdupq_n_f32(0.0f);
}
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
for(aa=0; aa<VEC1_COVAR_NACC_NEON; aa++){
float32x4_t vf = vld1q_f32(&aIn[vv][ii+4*aa]);
aAcc[aa] = vfmaq_f32(aAcc[aa], vs[vv], vf);
}
}
for(aa=0; aa<VEC1_COVAR_NACC_NEON; aa++){
vec1AddToDoubleArrayNEON(&aOut[ii+4*aa], aAcc[aa]);
}
}
for(; ii<=(nElem-4); ii+=4){
float32x4_t acc = vdupq_n_f32(0.0f);
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
float32x4_t vf = vld1q_f32(&aIn[vv][ii]);
acc = vfmaq_f32(acc, vs[vv], vf);
}
vec1AddToDoubleArrayNEON(&aOut[ii], acc);
}
#endif /* VEC1_HAVE_NEON */
for(; ii<nElem; ii++){
double acc = aOut[ii];
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
acc += (double)aScalar[vv] * aIn[vv][ii];
}
aOut[ii] = acc;
}
}
static void vec1CovarianceWork(void *pArg){
Vec1CovarianceJob *pJob = (Vec1CovarianceJob*)pArg;
const int iSubOff = pJob->iSub*pJob->nCodeElem;
float *aCode = &pJob->aBook[iSubOff * VEC1_PQ_CODEBOOK_SZ];
int ii, kk;
int nVec = pJob->pVec->nVec;
int nElem = pJob->pVec->nElem;
int nKK = MIN(pJob->nCodeElem, nElem - iSubOff);
#if defined(VEC1_USE_TRANSPOSITION)
/* Populate this job's transposed copy of its slice of the codebook. */
vec1CodebookTranspose(
aCode, pJob->nCodeElem, VEC1_NTRANSPOSE_WIDTH, VEC1_PQ_CODEBOOK_SZ,
pJob->aTrans
);
#endif
/* Loop through the training vectors that are our responsibility,
** VEC1_COVAR_NVEC at a time. Each iteration of this outer loop
** compares VEC1_COVAR_NVEC sub-vectors to all candidate codebook
** entries (VEC1_MULTIMATCH_NVEC at a time, as that is the width the
** best-match routines operate on), then updates pJob->aM[] with the
** results. */
ii = 0;
for(; ii<=(nVec-VEC1_COVAR_NVEC); ii+=VEC1_COVAR_NVEC){
int vv, iBatch;
const float *aVec[VEC1_COVAR_NVEC];
const float *aSub[VEC1_COVAR_NVEC];
int aBest[VEC1_COVAR_NVEC];
const float *aSubHat[VEC1_COVAR_NVEC];
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
aVec[vv] = vec1TrainingVector(pJob->pVec, ii+vv);
aSub[vv] = &aVec[vv][iSubOff];
}
for(iBatch=0; iBatch<VEC1_COVAR_MULTIMATCH_MULT; iBatch++){
int off = iBatch*VEC1_MULTIMATCH_NVEC;
vec1BestMatchPlatformN(
pJob->nCodeElem, aCode, pJob->aTrans, VEC1_PQ_CODEBOOK_SZ,
&aSub[off], &aBest[off], &pJob->fTotalDist
);
}
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
aSubHat[vv] = &aCode[aBest[vv] * pJob->nCodeElem];
}
/* For each row of pJob->aM[], gather the VEC1_COVAR_NVEC
** scalars that apply to it and stream through the whole row once,
** rather than once per training vector. */
for(kk=0; kk<nKK; kk++){
float aScalar[VEC1_COVAR_NVEC];
for(vv=0; vv<VEC1_COVAR_NVEC; vv++){
aScalar[vv] = aSubHat[vv][kk];
}
vec1CovarianceAccumulateN(
&pJob->aM[(kk+iSubOff) * nElem], aVec, aScalar, nElem
);
}
}
/* Scalar tail - handle training vectors not covered by the batched
** loop above because nVec is not a multiple of VEC1_COVAR_NVEC. */
for(; ii<nVec; ii++){
float *aVec = vec1TrainingVector(pJob->pVec, ii);
const float *aSub = &aVec[iSubOff];
int iBest = vec1BestMatchSimple(
aCode, VEC1_PQ_CODEBOOK_SZ, aSub, pJob->nCodeElem, &pJob->fTotalDist
);
float *aSubHat = &aCode[iBest * pJob->nCodeElem];
for(kk=0; kk<nKK; kk++){
float subhat_kk = aSubHat[kk];
double *aMCol = &pJob->aM[(kk+iSubOff) * nElem];
int iVec;
for(iVec=0; iVec<nElem; iVec++){
aMCol[iVec] += subhat_kk * aVec[iVec];
}
}
}
}
/*
** aVecTrain:
** Packed array of p->nVec vectors, each p->nElem elements each.
*/
static void vec1PqFindRotation(
Vec1TrainCtx *p,
Vec1JobQueue *pQueue,
Vec1TrainVectors *pTrain, /* Rotated vectors to train on */
float *aBook, /* Current codebook */
float *aRotation /* OUT: best rotation */
){
if( p->rc==SQLITE_OK ){
double *aM = 0;
double *aU = 0;
double *aVT = 0;
double *aS = 0;
double *aW = 0;
int ii;
int nElem2 = (p->tv.nElem * p->tv.nElem);
Vec1CovarianceJob *aJob = 0;
int nJob = 0;
/* Allocate space for 4 nElem*nElem matrices - W, M, U and VT. And one
** nElem vector - S. */
aM = (double*)vec1MallocZero(
4 * nElem2 * sizeof(double) +
p->tv.nElem * sizeof(double)
);
if( aM==0 ){
p->rc = SQLITE_NOMEM;
sqlite3_result_error_nomem(p->pCtx);
return;
}
aVT = &aM[nElem2];
aU = &aVT[nElem2];
aW = &aU[nElem2];
aS = &aW[nElem2];
/* Step 1. Calculate the cross-covariance matrix M between training set X
** and the quantized version of the training set X^. i.e. the sum of the
** following for all x in X:
**
** xT . x^
**
** (x1) (x1*x1^, x1*x2^, x1*x3^)
** (x2) . (x1^, x2^, x3^) == (x2*x1^, x2*x2^, x2*x3^)
** (x3) (x3*x1^, x3*x2^, x3*x3^)
**
** Because the covariance matrix is going to LAPACK, it has to be in
** column-major order. So aM[1]==(x2*x1^).
*/
START_TRAINING_TIMER(p, VEC1_TRAINING_ROT_COVARIANCE);
aJob = vec1CovarianceAlloc(p, aM, pTrain, aBook, &nJob);
if( aJob ){
for(ii=0; ii<nJob; ii++){
vec1JobQueueAddJob(pQueue, vec1CovarianceWork, 0, &aJob[ii]);
}
vec1JobQueueFinishJobs(pQueue, SQLITE_OK);
sqlite3_free(aJob);
}
END_TRAINING_TIMER(p, VEC1_TRAINING_ROT_COVARIANCE);
START_TRAINING_TIMER(p, VEC1_TRAINING_ROT_JACOBI);
memcpy(aW, aM, sizeof(double)*nElem2);
vec1JacobiSVD(p, pQueue, aW, p->tv.nElem, aU, aS, aVT, 50, 1e-14);
END_TRAINING_TIMER(p, VEC1_TRAINING_ROT_JACOBI);
vec1MatrixMultiply(aRotation, aU, aVT, p->tv.nElem);
if( p->rc==SQLITE_OK && p->bSvdVerify ){
p->rc = vec1SVDSelfTest(p->pCtx, aM, aU, aS, aVT, p->tv.nElem);
}
sqlite3_free(aM);
}
}
typedef struct Vec1RotationJob Vec1RotationJob;
struct Vec1RotationJob {
Vec1TrainVectors *pVec; /* Input vectors */
Vec1TrainVectors *pOut; /* Output vectors */
int iFirst;
int iEof;
const float *aRotation; /* nElem*nElem rotation matrix */
#ifdef VEC1_USE_TRANSPOSITION
const float *aRotationNT; /* Transposed rotation matrix */
#endif
};
static Vec1RotationJob *vec1RotationAlloc(
Vec1TrainCtx *p,
const float *aRotation,
const float *aRotationNT,
Vec1TrainVectors *pOut /* Store rotated vectors here */
){
int nByte = p->nThread * sizeof(Vec1RotationJob);
Vec1RotationJob *aJob = vec1MallocZero(nByte);
if( aJob ){
int ii;
for(ii=0; ii<p->nThread; ii++){
Vec1RotationJob *pJob = &aJob[ii];
ASSIGN_VECTORS_TO_JOB(&p->tv, pJob, ii, p->nThread);
pJob->aRotation = aRotation;
pJob->pOut = pOut;
#ifdef VEC1_USE_TRANSPOSITION
pJob->aRotationNT = aRotationNT;
#else
(void)aRotationNT;
#endif
}
}
return aJob;
}
#if defined(VEC1_USE_TRANSPOSITION)
static void vec1RotateVectorTransposeN(
float *aOut[VEC1_MULTIMATCH_NVEC],
const float *aIn[VEC1_MULTIMATCH_NVEC],
int nElem,
const float *aRotation,
const float *aRotationNT
){
int vv;
int d;
int ii;
/* ii is the first output element in the VEC1_MULTIMATCH_NVEC output
** vectors. Each iteration of this outer loop populates 8 elements
** of each of them */
for(ii=0; ii<=(nElem-8); ii+=8){
const float *aNT = &aRotationNT[(ii * nElem)];
__m256 aAcc[VEC1_MULTIMATCH_NVEC];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aAcc[vv] = _mm256_set1_ps(0.0f);
}
for(d=0; d<nElem; d++){
__m256 vec = LOADU( &aNT[ d * 8 ] );
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aAcc[vv] = FMADD(vec, _mm256_set1_ps(aIn[vv][d]), aAcc[vv] );
}
}
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
_mm256_storeu_ps(&aOut[vv][ii], aAcc[vv]);
}
}
for( ; ii<nElem; ii++){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aOut[vv][ii] = vec1DotProduct(nElem, aIn[vv], &aRotation[ii*nElem]);
}
}
}
#else
static void vec1RotateVectorN(
float *aOut[VEC1_MULTIMATCH_NVEC],
const float *aIn[VEC1_MULTIMATCH_NVEC],
int nElem,
const float *aRotation
){
int vv;
int d = 0;
#if defined(VEC1_HAVE_NEON)
/* Each iteration of this outer loop calculates values d..(d+3) for each
** of the VEC1_MULTIMATCH_NVEC vectors. */
for(; d<=(nElem-4); d+=4){
/* Accumulator aAcc[x][y] accumulates values such that its horizontal
** sum is the dot product of input vector aIn[x] and row (d+y) of
** the rotation matrix. */
const float *aRow[4];
int x, y, d2;
float32x4_t aAcc[VEC1_MULTIMATCH_NVEC][4];
for(y=0; y<4; y++) aRow[y] = &aRotation[(d+y)*nElem];
/* Zero all accumulators to start with. */
for(x=0; x<VEC1_MULTIMATCH_NVEC; x++){
for(y=0; y<4; y++) aAcc[x][y] = vdupq_n_f32(0.0f);
}
for(d2=0; d2<=(nElem-4); d2+=4){
float32x4_t aRot[4];
for(y=0; y<4; y++){
aRot[y] = vld1q_f32(&aRow[y][d2]);
}
for(x=0; x<VEC1_MULTIMATCH_NVEC; x++){
float32x4_t vec = vld1q_f32(&aIn[x][d2]);
for(y=0; y<4; y++){
aAcc[x][y] = vfmaq_f32(aAcc[x][y], vec, aRot[y]);
}
}
}
for(x=0; x<VEC1_MULTIMATCH_NVEC; x++){
/* Reduce the 4 accumulators for vector x to a single register
** containing output values d..(d+3). Then add in the scalar tail,
** if any. This is the same order of operations as vec1DotProduct(),
** so results are identical to those of vec1RotateVector(). */
float32x4_t res = vpaddq_f32(
vpaddq_f32(aAcc[x][0], aAcc[x][1]),
vpaddq_f32(aAcc[x][2], aAcc[x][3])
);
if( d2<nElem ){
float aTmp[4];
int k;
vst1q_f32(aTmp, res);
for(k=d2; k<nElem; k++){
for(y=0; y<4; y++) aTmp[y] += aIn[x][k] * aRow[y][k];
}
res = vld1q_f32(aTmp);
}
vst1q_f32(&aOut[x][d], res);
}
}
#endif
for(; d<nElem; d++){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aOut[vv][d] = vec1DotProduct(nElem, aIn[vv], &aRotation[d*nElem]);
}
}
}
#endif
static void vec1RotationWork(void *pArg){
Vec1RotationJob *pJob = (Vec1RotationJob*)pArg;
int nElem = pJob->pVec->nElem;
int ii = pJob->iFirst;
for(; ii<=(pJob->iEof-VEC1_MULTIMATCH_NVEC); ii+=VEC1_MULTIMATCH_NVEC){
const float *aIn[VEC1_MULTIMATCH_NVEC];
float *aOut[VEC1_MULTIMATCH_NVEC];
int vv;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aIn[vv] = vec1TrainingVector(pJob->pVec, vv+ii);
aOut[vv] = vec1TrainingVector(pJob->pOut, vv+ii);
}
#ifdef VEC1_USE_TRANSPOSITION
vec1RotateVectorTransposeN(
aOut, aIn, nElem, pJob->aRotation, pJob->aRotationNT
);
#else
vec1RotateVectorN(aOut, aIn, nElem, pJob->aRotation);
#endif
}
for(; ii<pJob->iEof; ii++){
const float *pIn = vec1TrainingVector(pJob->pVec, ii);
float *pOut = vec1TrainingVector(pJob->pOut, ii);
vec1RotateVector(nElem, pJob->aRotation, pIn, pOut);
}
}
/*
** Create and return a deep copy of the training vectors passed as the
** only argument.
*/
static Vec1TrainVectors *vec1CopyVectors(Vec1TrainVectors *pIn){
Vec1TrainVectors *pRet = 0;
int ii;
pRet = (Vec1TrainVectors*)vec1MallocZero(sizeof(Vec1TrainVectors));
if( pRet==0 ) return pRet;
pRet->nElem = pIn->nElem;
pRet->nVec = pIn->nVec;
pRet->nChunk = pIn->nChunk;
pRet->nVecPerChunk = pIn->nVecPerChunk;
for(ii=0; ii<pIn->nChunk; ii++){
int nByte = pRet->nVecPerChunk * pIn->nElem * sizeof_f32;
pRet->aChunk[ii] = (float*)sqlite3_malloc(VEC1_TRAINING_SZCHUNK);
if( pRet->aChunk[ii]==0 ){
vec1FreeVectors(pRet);
return 0;
}
if( ii==pIn->nChunk-1 ){
nByte = (pRet->nVec - ii*pRet->nVecPerChunk) * pRet->nElem*sizeof_f32;
}
assert( nByte<=VEC1_TRAINING_SZCHUNK );
memcpy(pRet->aChunk[ii], pIn->aChunk[ii], nByte);
}
return pRet;
}
static void vec1TrainRotation(
Vec1TrainCtx *p,
Vec1JobQueue *pQueue,
float *aBook,
float *aRotation
){
const int nElem = p->tv.nElem;
const int nMatrixByte = sizeof_f32 * nElem * nElem;
int iRound = 0; /* Current OPQ round */
int ii;
float *aRot = (float*)sqlite3_malloc(nMatrixByte * 2);
float *aRot2 = &aRot[nElem*nElem];
Vec1TrainVectors *pTrain = vec1CopyVectors(&p->tv);
Vec1RotationJob *aJob = 0;
aJob = vec1RotationAlloc(p, aRotation, aRot2, pTrain);
if( pTrain==0 || aRot==0 || aJob==0 ){
sqlite3_result_error_nomem(p->pCtx);
p->rc = SQLITE_NOMEM;
}
for(iRound=0; iRound<p->nOpqRound; iRound++){
vec1TrainLog(p, "OPQ round %d/%d", iRound+1, p->nOpqRound);
START_TRAINING_TIMER(p, VEC1_TRAINING_ROT_CODEBOOKS);
vec1TrainPQCodebooks(p, pTrain, pQueue, aBook);
END_TRAINING_TIMER(p, VEC1_TRAINING_ROT_CODEBOOKS);
vec1PqFindRotation(p, pQueue, pTrain, aBook, aRot);
if( p->rc==SQLITE_OK ){
if( iRound==0 ){
memcpy(aRotation, aRot, nMatrixByte);
}else{
memcpy(aRot2, aRotation, nMatrixByte);
vec1MatrixMultiplyF(aRotation, aRot2, aRot, nElem);
}
START_TRAINING_TIMER(p, VEC1_TRAINING_ROT_VECTORS);
#ifdef VEC1_USE_TRANSPOSITION
vec1CodebookTranspose(aRotation,
nElem, VEC1_NTRANSPOSE_WIDTH, nElem, aRot2
);
#endif
for(ii=0; ii<p->nThread; ii++){
vec1JobQueueAddJob(pQueue, vec1RotationWork, 0, &aJob[ii]);
}
vec1JobQueueFinishJobs(pQueue, SQLITE_OK);
END_TRAINING_TIMER(p, VEC1_TRAINING_ROT_VECTORS);
}
p->nCompletedWork += VEC1_TRAINING_WORK_OPQ;
}
if( p->rc==SQLITE_OK ){
for(ii=0; ii<p->tv.nChunk; ii++){
SWAP(float*, p->tv.aChunk[ii], pTrain->aChunk[ii]);
}
}
sqlite3_free(aJob);
sqlite3_free(aRot);
vec1FreeVectors(pTrain);
}
static void vec1TrainLogTimes(Vec1TrainCtx *p){
double fTotal = (double)p->aTime[VEC1_TRAINING_TOTAL];
if( p->zProfileFunction ){
struct Counter {
const char *zName;
int iTime;
} aCounter[] = {
{ "rot_codebooks", VEC1_TRAINING_ROT_CODEBOOKS },
{ "rot_covariance", VEC1_TRAINING_ROT_COVARIANCE },
{ "rot_jacobi", VEC1_TRAINING_ROT_JACOBI },
{ "rot_vectors", VEC1_TRAINING_ROT_VECTORS },
{ "coarse_init", VEC1_TRAINING_COARSE_INIT },
{ "coarse_lloyds", VEC1_TRAINING_COARSE_KMEANS },
{ "residuals", VEC1_TRAINING_RESIDUALS },
{ "codebooks", VEC1_TRAINING_CODEBOOKS }
};
int icall;
for(icall=0; icall<2; icall++){
int ii;
char *zSql = 0;
sqlite3_str *pSql = sqlite3_str_new(0);
sqlite3_str_appendf(pSql, "SELECT \"%w\"('{", p->zProfileFunction);
if( icall==0 ){
for(ii=0; ii<size_of_array(aCounter); ii++){
sqlite3_str_appendf(pSql, "%s\"%s\": %.2f", ((ii>0) ? ", " : ""),
aCounter[ii].zName, p->aTime[ aCounter[ii].iTime ]*100.00 / fTotal
);
}
}else{
for(ii=0; ii<size_of_array(aCounter); ii++){
u64 val = p->aTime[ aCounter[ii].iTime ];
sqlite3_str_appendf(pSql, "%s\"%s\": ",
((ii>0) ? ", " : ""), aCounter[ii].zName
);
if( val>(1000*1000*1000) ){
sqlite3_str_appendf(pSql, "\"%.2fG\"", (double)val / 1000000000.0);
}else if( val>(10*1000*1000) ){
sqlite3_str_appendf(pSql, "\"%dM\"", (int)(val / (1000*1000)));
}else if( val>(1*1000*1000) ){
sqlite3_str_appendf(pSql, "\"%.2fM\"", (double)val / 1000000.0);
}else if( val>(10*1000) ){
sqlite3_str_appendf(pSql, "\"%dK\"", (int)(val / (1000)));
}else if( val>(1000) ){
sqlite3_str_appendf(pSql, "\"%.2fK\"", (double)val / 1000.0);
}else{
sqlite3_str_appendf(pSql, "\"%d\"", (int)val);
}
}
}
sqlite3_str_appendf(pSql, "}')");
zSql = sqlite3_str_finish(pSql);
if( zSql==0 ){
sqlite3_result_error_nomem(p->pCtx);
break;
}else{
sqlite3 *db = sqlite3_context_db_handle(p->pCtx);
char *zErrmsg = 0;
int rc = sqlite3_exec(db, zSql, 0, 0, &zErrmsg);
if( rc!=SQLITE_OK ){
sqlite3_result_error(p->pCtx, zErrmsg, -1);
sqlite3_result_error_code(p->pCtx, rc);
break;
}
sqlite3_free(zErrmsg);
sqlite3_free(zSql);
}
}
}
}
typedef struct Vec1ResidualJob Vec1ResidualJob;
struct Vec1ResidualJob {
/* Data to train on */
const Vec1TrainCtx *p;
Vec1TrainVectors *pVec;
int iFirst;
int iEof;
const float *aCentroid;
int nCentroid;
};
static void vec1TrainResidualWorker(void *pCtx){
Vec1ResidualJob *p = (Vec1ResidualJob*)pCtx;
int ii;
int nElem = p->pVec->nElem;
for(ii=p->iFirst; ii<p->iEof; ii++){
float *a = vec1TrainingVector(p->pVec, ii);
int iBest = vec1BestMatchSimple(p->aCentroid, p->nCentroid, a, nElem, 0);
vec1SubInPlace(a, &p->aCentroid[iBest * nElem], nElem);
}
}
static void vec1TrainFindResiduals(
Vec1TrainCtx *p,
Vec1JobQueue *pQueue,
const float *aCentroid
){
if( p->rc==SQLITE_OK ){
int ii;
int nJob = p->nThread;
Vec1ResidualJob *aJob = 0;
int rc = SQLITE_OK;
assert( nJob>=1 );
aJob = (Vec1ResidualJob*)vec1MallocZero(sizeof(Vec1ResidualJob) * nJob);
if( aJob==0 ){
sqlite3_result_error_nomem(p->pCtx);
p->rc = SQLITE_NOMEM;
return;
}
for(ii=0; ii<nJob; ii++){
Vec1ResidualJob *pJob = &aJob[ii];
ASSIGN_VECTORS_TO_JOB(&p->tv, pJob, ii, nJob);
pJob->p = p;
pJob->aCentroid = aCentroid;
pJob->nCentroid = p->nBucket;
vec1JobQueueAddJob(pQueue, vec1TrainResidualWorker, 0, (void*)pJob);
}
vec1JobQueueFinishJobs(pQueue, rc);
sqlite3_free(aJob);
}
}
static void vec1TrainPrepareLog(Vec1TrainCtx *p){
if( p->zLogFunction ){
char *zSql = sqlite3_mprintf("SELECT %s(?, ?)", p->zLogFunction);
if( zSql==0 ){
sqlite3_result_error_nomem(p->pCtx);
p->rc = SQLITE_NOMEM;
}else{
sqlite3 *db = sqlite3_context_db_handle(p->pCtx);
p->rc = sqlite3_prepare_v2(db, zSql, -1, &p->pLogStmt, 0);
sqlite3_free(zSql);
if( p->rc==SQLITE_ERROR ){
sqlite3_result_error(p->pCtx, sqlite3_errmsg(db), -1);
}else if( p->rc!=SQLITE_OK ){
sqlite3_result_error_code(p->pCtx, p->rc);
}
}
}
}
static void vec1TrainWorkInit(Vec1TrainCtx *p){
int nTotal = 0;
if( p->eQuant==VEC1_QUANTIZE_OPQ ){
nTotal += VEC1_TRAINING_WORK_OPQ * p->nOpqRound;
}
if( p->nBucket>0 ){
nTotal += VEC1_TRAINING_WORK_COARSE1 + VEC1_TRAINING_WORK_COARSE2;
}
if( p->eQuant==VEC1_QUANTIZE_PQ || p->eQuant==VEC1_QUANTIZE_OPQ ){
nTotal += VEC1_TRAINING_WORK_CODEBOOKS;
}
p->nTotalWork = (nTotal ? nTotal : 1);
}
/*
** Return true if n is a power of 2, or false otherwise.
*/
static int vec1IsPow2(int n) {
return n > 0 && (n & (n-1)) == 0;
}
/*
** Apply Walsh-Hadamard Transform to n element vector x. n must
** be a power of 2.
*/
static void vec1WHT(float *x, int n){
assert( vec1IsPow2(n) );
for (int stride = 1; stride < n; stride <<= 1) {
for (int i = 0; i < n; i += stride << 1) {
for (int j = i; j < i + stride; j++) {
float a = x[j];
float b = x[j + stride];
x[j] = a + b;
x[j+stride] = a - b;
}
}
}
}
/*
** Apply Walsh-Hadamard transform to nElem element vector aVec.
*/
static void vec1ApplyWHT(float *aVec, int nElem){
if (vec1IsPow2(nElem)) {
vec1WHT(aVec, nElem);
}else{
int p = 1;
while( (p<<1)<nElem ) p = p<<1;
vec1WHT(aVec, p);
vec1WHT(aVec + nElem - p, p);
}
}
/*
** Apply the Walsh-Hadamard Transform to all training vectors used by
** the training context.
*/
static void vec1TrainApplyWHT(Vec1TrainCtx *p){
int ii;
for(ii=0; ii<p->tv.nVec; ii++){
vec1ApplyWHT(vec1TrainingVector(&p->tv, ii), p->tv.nElem);
}
}
/*
** The xFinal() method for the vec1_train() aggregate function.
**
** Output a trained model blob.
*/
static void vec1TrainFinal(sqlite3_context *pCtx){
Vec1TrainCtx *p = 0;
float *aBook = 0; /* Code books */
float *aCentroid = 0; /* Centroids */
float *aRotation = 0; /* Rotation */
int nByte = 0;
u8 *aByte = 0;
float *pCsr = 0;
Vec1JobQueue *pQueue = 0;
u32 flags = 0;
int ii;
int bPQ = 0;
/* Check that sufficient training vectors were provided. */
p = (Vec1TrainCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p));
if( p==0 ) return;
p->pCtx = pCtx;
bPQ = (p->eQuant==VEC1_QUANTIZE_PQ || p->eQuant==VEC1_QUANTIZE_OPQ);
if( p->tv.nVec==0 ){
vec1ResultErrorF(pCtx,
"too few training vectors (have 0, require more than that)"
);
goto train_final_out;
}
if( p->nBucket || bPQ ){
int nReq = p->nBucket * 4;
if( bPQ ) nReq = MAX(VEC1_PQ_TRAINING_SET_MIN, nReq);
if( p->tv.nVec<nReq ){
vec1ResultErrorF(pCtx,
"too few training vectors (require %d, have %d)",
nReq, p->tv.nVec
);
goto train_final_out;
}
}
vec1TrainWorkInit(p);
vec1TrainPrepareLog(p);
vec1TrainLog(p, "vectors loaded, starting training");
if( p->rc!=SQLITE_OK ) goto train_final_out;
START_TRAINING_TIMER(p, VEC1_TRAINING_TOTAL);
/* If this is training for "cos" distance, not "l2", normalize all
** training vectors before proceding as usual. */
if( p->eDistance==VEC1_DISTANCE_COS ){
vec1TrainNormalizeAll(p);
}
/* Allocate space for the model. Space is required for (A) the header and
** coarse centroids, (B) the PQ codebooks and (C) the OPQ rotation
** matrix. */
nByte = VEC1_HEADER_SIZE + (sizeof_f32 * p->nBucket * p->tv.nElem); /* A */
if( bPQ ){
nByte += sizeof_f32*p->nCodesize*p->nCodeElem*VEC1_PQ_CODEBOOK_SZ; /* B */
}
if( p->eQuant==VEC1_QUANTIZE_OPQ ){
nByte += sizeof_f32 * p->tv.nElem * p->tv.nElem; /* C */
}
aByte = (u8*)vec1ContextMalloc(pCtx, nByte);
if( aByte==0 ) goto train_final_out;
pCsr = (float*)&aByte[VEC1_HEADER_SIZE];
if( bPQ ){
aBook = pCsr;
pCsr += (p->nCodesize * p->nCodeElem * VEC1_PQ_CODEBOOK_SZ);
}
if( p->nBucket>0 ){
aCentroid = pCsr;
pCsr += (p->nBucket * p->tv.nElem);
}
if( p->eQuant==VEC1_QUANTIZE_OPQ ){
aRotation = pCsr;
pCsr += (p->tv.nElem * p->tv.nElem);
}
assert( (u8*)pCsr==&aByte[nByte] );
/* Allocate a jobs queue, if multi-threading is required. If it is not
** required, leave pQueue set to NULL. Things will proceed in a single
** thread in that case. */
#if VEC1_THREADS
if( p->nThread>1 ){
pQueue = vec1JobQueueNew(p->nThread-1);
if( pQueue==0 ){
sqlite3_result_error_nomem(pCtx);
goto train_final_out;
}
}
#endif
/* Write the model header into the output buffer. */
flags = VEC1_MODEL_INDEX
| ((p->nBucket>0 && p->bResidual) ? VEC1_MODEL_RESIDUAL : 0)
| ((p->eQuant==VEC1_QUANTIZE_OPQ) ? VEC1_MODEL_ROTATE : 0)
| (p->bWHT ? VEC1_MODEL_WHT : 0)
| ((p->eQuant==VEC1_QUANTIZE_BQ) ? VEC1_MODEL_BITQUANT : 0);
vec1HeaderWrite(aByte, flags,
p->tv.nElem, p->nCodesize, p->nBucket, p->eDistance
);
/* If an OPQ rotation is require, calculate one now */
if( p->eQuant==VEC1_QUANTIZE_OPQ ){
vec1TrainRotation(p, pQueue, aBook, aRotation);
}else if( p->bWHT ){
vec1TrainApplyWHT(p);
}
if( p->nBucket>0 ){
vec1TrainCoarseQuant(p, pQueue, aCentroid);
/* For each training vector, find the closest centroid. Then subtract
** the centroid from the training vector, to leave the residual vector.
** This is what the PQ step trains on. */
if( bPQ && p->bResidual ){
vec1TrainLog(p, "calculating residuals");
START_TRAINING_TIMER(p, VEC1_TRAINING_RESIDUALS);
vec1TrainFindResiduals(p, pQueue, aCentroid);
END_TRAINING_TIMER(p, VEC1_TRAINING_RESIDUALS);
}
}
/* Now train the PQ codebooks, if required. Each codebook is trained in
** parallel, using a separate thread if threads were configured. */
if( bPQ ){
vec1TrainLog(p, "PQ codebook training");
START_TRAINING_TIMER(p, VEC1_TRAINING_CODEBOOKS);
vec1TrainPQCodebooks(p, &p->tv, pQueue, aBook);
p->nCompletedWork += VEC1_TRAINING_WORK_CODEBOOKS;
END_TRAINING_TIMER(p, VEC1_TRAINING_CODEBOOKS);
}
/* Return the result blob. */
if( p->rc==SQLITE_OK ){
sqlite3_result_blob(pCtx, (void*)aByte, nByte, vec1SqliteFree);
aByte = 0;
}
train_final_out:
END_TRAINING_TIMER(p, VEC1_TRAINING_TOTAL);
vec1TrainLog(p, "training complete");
vec1TrainLogTimes(p);
sqlite3_finalize(p->pLogStmt);
sqlite3_free(p->zLogFunction);
sqlite3_free(p->zProfileFunction);
for(ii=0; ii<p->tv.nChunk; ii++){
sqlite3_free(p->tv.aChunk[ii]);
}
vec1JobQueueFree(pQueue);
sqlite3_free(aByte);
}
typedef struct Vec1Model Vec1Model;
struct Vec1Model {
Vec1ModelHeader hdr;
const float *aModel;
const float *aCentroid;
const float *aRotation;
int nCodeElem;
float *aModelT;
#ifdef VEC1_USE_TRANSPOSITION
float *aCentroidNT;
float *aModelNT;
float *aRotationNT;
#endif
};
static int vec1DecodeModel(
const u8 *aBlob,
int nByte,
Vec1Model *pMod,
char **pzErr
){
const u8 *pCsr = aBlob;
int nCodebook = 0;
if( nByte<VEC1_HEADER_SIZE ){
*pzErr = sqlite3_mprintf("vec1: model too small - %d bytes", nByte);
return SQLITE_ERROR;
}
vec1HeaderRead(aBlob, &pMod->hdr);
pCsr += VEC1_HEADER_SIZE;
assert( VEC1_HEADER_VERSION_BIN>VEC1_HEADER_VERSION );
if( pMod->hdr.iVersion!=VEC1_HEADER_VERSION
&& pMod->hdr.iVersion!=VEC1_HEADER_VERSION_BIN
){
if( pMod->hdr.iVersion<VEC1_HEADER_VERSION_BIN ){
*pzErr = sqlite3_mprintf("vec1: unrecognized model version");
}else{
*pzErr = sqlite3_mprintf(
"vec1: unsupported model version - require vec1 %d.%d",
(pMod->hdr.iVersion / 1000), (pMod->hdr.iVersion % 1000)
);
}
return SQLITE_ERROR;
}
/* Check header fields are within the acceptable ranges. This also serves
** to ensure that the 32-bit arithmetic used to calculate offsets below
** does not overflow. */
if( pMod->hdr.nElem>VEC1_VECSIZE_MAX
|| pMod->hdr.nBucket>VEC1_MAX_BUCKET
|| pMod->hdr.nCodebook>pMod->hdr.nElem
){
*pzErr = sqlite3_mprintf("vec1: model is corrupt");
return SQLITE_ERROR;
}
nCodebook = pMod->hdr.nCodebook;
if( nCodebook>0 ){
if( (pMod->hdr.flags & VEC1_MODEL_BITQUANT)==0 ){
pMod->nCodeElem = (pMod->hdr.nElem + nCodebook-1)/nCodebook;
pMod->aModel = (float*)pCsr;
pCsr += (nCodebook*pMod->nCodeElem*VEC1_PQ_CODEBOOK_SZ * sizeof_f32);
}else{
if( ((pMod->hdr.nElem + 7)/8)==(u32)nCodebook ){
pMod->nCodeElem = 8;
}else if( ((pMod->hdr.nElem + 15)/16)==(u32)nCodebook ){
pMod->nCodeElem = 16;
}else{
*pzErr = sqlite3_mprintf("vec1: unrecognized bq codesize");
return SQLITE_ERROR;
}
}
}
if( pMod->hdr.nBucket>0 ){
pMod->aCentroid = (float*)pCsr;
pCsr += (pMod->hdr.nElem * pMod->hdr.nBucket * sizeof_f32);
}
if( (pMod->hdr.flags & VEC1_MODEL_ROTATE) ){
pMod->aRotation = (float*)pCsr;
pCsr += (pMod->hdr.nElem * pMod->hdr.nElem * sizeof_f32);
}
if( (pCsr - aBlob)!=nByte ){
*pzErr = sqlite3_mprintf(
"vec1: model size mismatch: have %d, expected %d", nByte, (pCsr-aBlob)
);
return SQLITE_ERROR;
}
return SQLITE_OK;
}
#if defined(VEC1_USE_TRANSPOSITION)
static int vec1ModelAllocTCent(Vec1Model *pMod){
int rc = SQLITE_OK;
if( pMod->aCentroidNT==0 && (pMod->hdr.nBucket>0 || pMod->aModel) ){
int nElem = pMod->hdr.nElem;
int nByte = sizeof_f32 * nElem * pMod->hdr.nBucket;
if( pMod->aModel ){
int nFloatPerCodebook = pMod->nCodeElem * pMod->hdr.nCodebook;
nByte += (VEC1_PQ_CODEBOOK_SZ * nFloatPerCodebook * sizeof_f32);
if( pMod->aRotation ){
nByte += (nElem * nElem) * sizeof_f32;
}
}
pMod->aCentroidNT = (float*)sqlite3_malloc(nByte);
if( pMod->aCentroidNT==0 ){
rc = SQLITE_NOMEM;
}else{
assert( pMod->aCentroid || pMod->hdr.nBucket==0 );
vec1CodebookTranspose(pMod->aCentroid, nElem,
VEC1_NTRANSPOSE_WIDTH, pMod->hdr.nBucket, pMod->aCentroidNT
);
if( pMod->aModel ){
u32 M;
int szCodebook = pMod->nCodeElem * VEC1_PQ_CODEBOOK_SZ;
pMod->aModelNT = &pMod->aCentroidNT[nElem * pMod->hdr.nBucket];
for(M=0; M<pMod->hdr.nCodebook; M++){
vec1CodebookTranspose(
&pMod->aModel[szCodebook*M],
pMod->nCodeElem, VEC1_NTRANSPOSE_WIDTH, VEC1_PQ_CODEBOOK_SZ,
&pMod->aModelNT[szCodebook*M]
);
}
if( pMod->aRotation ){
/* Create a transposed version of the rotation matrix. */
pMod->aRotationNT = &pMod->aModelNT[szCodebook * M];
vec1CodebookTranspose(pMod->aRotation,
nElem, VEC1_NTRANSPOSE_WIDTH, nElem, pMod->aRotationNT
);
}
}
}
}
return rc;
}
static void vec1ModelFreeTCent(Vec1Model *pMod){
sqlite3_free(pMod->aCentroidNT);
pMod->aCentroidNT = 0;
pMod->aModelNT = 0;
pMod->aRotationNT = 0;
}
#else
# define vec1ModelAllocTCent(pMod) SQLITE_OK
# define vec1ModelFreeTCent(pMod)
#endif
/*************************************************************************
** Start of virtual table code
*/
/* %_IDX TABLE LIST FORMAT:
**
** Each list begins with 3 32-bit unsigned integers in big-endian format.
** As follows:
**
** bytes 0..3: Flags value.
** bytes 4..7: Total number of entries in list.
** bytes 8..11: Number of tombstone entries in list
**
** Following these is an array of rowid values - either unsigned 32-bit
** values or 64-bit signed values, depending on the flags set in the
** first field value of the list. In the 32-bit format, the tombstone
** value is 0xFFFFFFFF. For the 64-bit format, it is 0. Rowids are stored
** in big-endian format.
**
** There are two formats - one that uses 32-bit rowids, and another that
** uses 64-bit rowids. Format consists of one or more blocks. Each block
** stores 16 PQ codes:
**
** * 16 rowid values - either 16 * 32-bit unsigned or 16 * 64-bit
** signed. The only way to know which is to read the first/last
** column values from the %_idx table.
**
** * The lookup value for the first codebook for each of the 16
** codes in the block. Followed by the lookup value for the second
** codebook for each of the 16 codes, and so on.
**
** A PQ code list always consists of an exact number of blocks. In the
** 32-bit rowid format, a rowid value of 0xFFFFFFFF indicates an invalid
** entry. In the 64-bit format, an invalid entry is one with a rowid value
** of 0.
*/
/* %_META TABLE LIST FORMAT:
**
** Meta table lists always begin with 2 32-bit unsigned integers in
** big-endian format:
**
** Bytes 0..3: 32-bit big-endian flags field.
** Bytes 4..7: 32-bit big-endian nEntry field.
**
** The flags field is set to a combination of VEC1_META_XXX flags. The
** nEntry fields is set to the number of entries in the list. The format
** of the rest of the list is different depending on the flags:
**
** VEC1_META_GENERIC:
**
** This is the fallback format used if none of the other formats can be
** used. It encodes each value separately as a varint type followed by
** encoded data. Specifically, for type values 0, 1 and 2:
**
** 0: NULL value. 0 byte payload.
** 1: Unsigned integer value in range [0..254]. 1 byte payload.
** 2: Integer value. 4 byte payload in range [-2147483647..+2147483647].
** 3: Integer value. 8 byte payload.
** 4: Real value. 8 byte payload.
**
** Odd values larger than 2 are text values, with size in bytes of
** (type - 2)/2. Even values larger than 2 are blobs.
**
** VEC1_META_1BYTEINT:
** This format may be used if all values in the list are either NULL or
** integers between 0 and 254, inclusive. Each value is stored as
** a 1 byte unsigned value, with 255 (0xFF) representing NULL.
**
** VEC1_META_4BYTEINT:
** This format may be used if all values in the list are either NULL or
** integers between -2147483648 and +2147483646, inclusive. Each value is
** stored as a 4 byte two's complement value, with +2147483647
** (0x7FFFFFFF) representing NULL.
**
** VEC1_META_REAL:
** This format may be used if all values in the list are real or NULL.
** Bit pattern 0x7FF8000000000001 is used for NULL.
**
** Flag VEC1_META_HASNULL is set if the list contains at least one NULL
** value.
*/
#define VEC1_META_GENERIC 0x01
#define VEC1_META_1BYTEINT 0x02
#define VEC1_META_4BYTEINT 0x04
#define VEC1_META_REAL 0x08
#define VEC1_META_HASNULL 0x10
#define VEC1_META_TYPEMASK ( \
VEC1_META_GENERIC | VEC1_META_1BYTEINT | VEC1_META_4BYTEINT | VEC1_META_REAL \
)
#define VEC1_META_SZHDR 8
#define VEC1_META_1BYTENULL 255
#define VEC1_META_REALNULL 0x7FF8000000000001
#define VEC1_META_4BYTEMIN (-2147483647-1)
#define VEC1_META_4BYTEMAX +2147483646
#define VEC1_META_4BYTENULL +2147483647
/* When a meta-value array is loaded from disk, at least VEC1_META_PADDING
** extra bytes of memory are allocated. This is to allow readers to safely
** overread the record slightly if it is corrupt. */
#define VEC1_META_PADDING 20
typedef struct Vec1Tab Vec1Tab;
typedef struct Vec1Csr Vec1Csr;
/*
** Flags that may be part of the first 32-bit unsigned integer of a PQ list.
*/
#define VEC1_LIST_64BIT 0x01 /* Set for 64-bit rowids, clear for 32-bit */
#define VEC1_LIST_SORTED 0x02 /* True if rowid array is in sorted order */
/* Size of list header in byte */
#define VEC1_LIST_SZHDR (3*sizeof_u32)
#define VEC1_TOMBSTONE_32 0xFFFFFFFF
#define VEC1_TOMBSTONE_64 0
/*
** Fields in the virtual table:
*/
#define VEC1_COLUMN_CMD 0
#define VEC1_COLUMN_ARG 1
#define VEC1_COLUMN_DISTANCE 2
#define VEC1_COLUMN_VECTOR 3
/* Followed by Vec1Tab.nMeta meta-data columns */
/*
** MODEL (0):
** This integer value is set to 1 when a model is first provided to
** the table, and incremented each time the model is updated thereafter.
**
** BLOCKSIZE (1):
** The config entry for the target size (in bytes) for chunks stored in
** the %_idx table.
**
** FMTVERSION (5):
** This is an integer - the current format version. Currently this is
** always 1, but will be increased if the format changes. If a client
** finds a format it does not understand, it has to rebuild the index.
*/
#define VEC1_CONFIG_MODEL 0
#define VEC1_CONFIG_BLOCKSIZE 1
#define VEC1_CONFIG_NELEM 2
#define VEC1_CONFIG_BLOCKSIZE_MIN 3
#define VEC1_CONFIG_FMTVERSION 5
/*
** Return the shadow-schema used by the vec1_ann table named zTab. Parameter
** zDb must be the name of the database (e.g. "main", "temp" or the name
** of an attached database).
*/
static char *vec1ShadowSchema(const char *zDb, const char *zTab, int nMeta){
const char zFmt[] =
/* Configuration table. This contains persistent configuration parameters,
** including the training data. Keys must be one of the VEC1_CONFIG_XXX
** values defined above. */
"CREATE TABLE %Q.'%q_config'(id INTEGER PRIMARY KEY, val ANY);"
/* Base table. This contains the raw data as provided by the user. */
"CREATE TABLE %Q.'%q_base'("
"id INTEGER PRIMARY KEY, "
"vector BLOB%z"
");"
/* Index data */
"CREATE TABLE %Q.'%q_idx'("
"id INTEGER PRIMARY KEY, "/* Id of blob */
"bucket INTEGER, " /* Index of bucket for this blob (or NULL) */
"first INTEGER, " /* Smallest rowid in blob */
"last INTEGER, " /* Largest rowid in blob */
"val BLOB" /* Blob containing rowids + PQ codes */
");"
"CREATE INDEX %Q.'%q_idx_idx' ON '%q_idx'(bucket, first, last);"
/* Training data (codebooks and centroids). This is stored as a single
** blob with id=1, in the same format as returned by vec1_train(). */
"CREATE TABLE %Q.'%q_model'(id INTEGER PRIMARY KEY, val BLOB);"
/* Meta table. Contains arrays of meta values stored in parallel with
** arrays in the %_idx table. For column iMeta and %_idx array id=iId,
** the id used in this table is ((iId<<8) + iMeta). */
"CREATE TABLE %Q.'%q_meta'("
"id INTEGER PRIMARY KEY, "
"val BLOB"
");"
;
int ii;
char *zMeta = 0;
for(ii=0; ii<nMeta; ii++){
zMeta = sqlite3_mprintf("%z, c%d", zMeta, ii);
if( zMeta==0 ) return 0;
}
return sqlite3_mprintf(zFmt,
zDb, zTab, zDb, zTab,
zMeta,
zDb, zTab,
zDb, zTab, zTab,
zDb, zTab,
zDb, zTab /* CREATE TABLE %_meta */
);
}
static char *vec1DropShadowSchema(const char *zDb, const char *zTab){
const char zFmt[] =
"DROP TABLE IF EXISTS %Q.'%q_config';"
"DROP TABLE IF EXISTS %Q.'%q_base';"
"DROP TABLE IF EXISTS %Q.'%q_idx';"
"DROP TABLE IF EXISTS %Q.'%q_model';"
"DROP TABLE IF EXISTS %Q.'%q_meta';"
;
return sqlite3_mprintf(zFmt,
zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
);
}
typedef struct Vec1Config Vec1Config;
struct Vec1Config {
i64 iModelVersion;
int nBlocksize;
int nBlocksizeMin;
int nElem;
};
/*
** The number of PQ codes grouped together.
*/
#define VEC1_PQ_BLOCKSIZE 16
/*
** flags:
** This is a mask of the VEC1_META_XYZ flags. However, each flag
** is used slightly differently than in the mask in the header of
** a meta-value list. Specifically:
**
** 1. VEC1_META_1BYTEINT, VEC1_META_4BYTEINT and VEC1_META_REAL are
** each set if the current contents of the array does not preclude
** it from using that form (VEC1_META_GENERIC is always implicitly
** set in this sense).
**
** 2. VEC1_META_HASNULL is set if the array contains at least one
** NULL value.
**
** format:
** This is set to either VEC1_META_GENERIC, 1BYTEINT, 4BYTEINT or REAL.
** To indicate the actual format currently stored in buffer buf.
*/
typedef struct Vec1MetaBuilder Vec1MetaBuilder;
struct Vec1MetaBuilder {
Vec1Buffer buf;
u32 flags;
u32 format;
};
typedef struct Vec1ListBuilder Vec1ListBuilder;
struct Vec1ListBuilder {
Vec1Tab *pTab; /* Table this object belongs to */
int nVectorSize; /* Size of [compressed] vectors in bytes */
int iBucket; /* Value for "bucket" column */
int eBlocked; /* VEC1_BLOCKED_XYZ value */
i64 iId; /* Rowid for %_idx, or 0 for NULL */
i64 iFirst; /* Smallest rowid */
i64 iLast; /* Largest rowid */
int bSorted; /* True if rowids are sorted */
int szRowid; /* Size of rowids (4 or 8) */
int nTombstone; /* Current number of tombstone entries */
Vec1Buffer bufRowid; /* Buffer containing rowids so far */
Vec1Buffer bufData; /* Buffer containing data so far */
Vec1MetaBuilder *aMeta; /* Array of pTab->nMeta meta-list builders */
};
#define VEC1_BLOCKED_NONE 0
#define VEC1_BLOCKED_PQ 1
#define VEC1_BLOCKED_BIN 2
/*
** An object of this type is used to accumulate one or more new vector
** index entries in memory before they are flushed to disk.
*/
typedef struct Vec1Writer Vec1Writer;
struct Vec1Writer {
Vec1Tab *pTab; /* Vec1 table object */
int bRebuild; /* True if this is a 'rebuild' op */
float *aResidual; /* Temp space for residual vector */
u8 *aPQ; /* Temp space for PQ */
int nBld; /* Size of aBld[] */
Vec1ListBuilder *aBld; /* Array of list builders */
};
#define VEC1_INSTRUMENT_QUERIES 1
#ifdef VEC1_INSTRUMENT_QUERIES
# define VEC1_QINSTR_TOTAL 0
# define VEC1_QINSTR_INITQUERY 1
# define VEC1_QINSTR_COARSE 2
# define VEC1_QINSTR_IDXREAD 3
# define VEC1_QINSTR_METAREAD 4
# define VEC1_QINSTR_METASCAN 5
# define VEC1_QINSTR_LUT 6
# define VEC1_QINSTR_PQSCAN 7
# define VEC1_QINSTR_FINALSORT 8
# define VEC1_QINSTR_NCOUNTER 9
# define VEC1_QINSTR_START(pTab, eCounter) { \
(pTab)->aCycle[eCounter] -= vec1HardwareTimer(); \
}
# define VEC1_QINSTR_STOP(pTab, eCounter) { \
(pTab)->aCount[eCounter]++; \
(pTab)->aCycle[eCounter] += vec1HardwareTimer(); \
}
#else
# define VEC1_QINSTR_STOP(pTab, eCounter)
# define VEC1_QINSTR_START(pTab, eCounter)
#endif
#define VEC1_META_COLUMN_BITS 8
#define VEC1_MAX_META_COLUMNS (1<<VEC1_META_COLUMN_BITS)
/* Virtual table object */
struct Vec1Tab {
sqlite3_vtab base;
sqlite3 *db; /* Database handle */
char *zDb; /* Database containing vtab */
char *zName; /* Name of vtab */
char *zTrainTbl; /* Name of %_model table */
char *zIdxTbl; /* Name of %_idx table */
int nMeta; /* Number of meta columns */
sqlite3_stmt *aStmt[21];
Vec1Config cfg; /* Values read from %_config table */
Vec1Model mod; /* Model read from %_model table */
Vec1Writer *pWriter; /* Writer within active transaction */
float *aTmpVec; /* Temp buffer for transformed vector */
int nTmpVec; /* Size of aTmpVector in elements */
void *pModBlob; /* Pointer to blob to free */
#ifdef VEC1_INSTRUMENT_QUERIES
u64 aCycle[VEC1_QINSTR_NCOUNTER];
u64 aCount[VEC1_QINSTR_NCOUNTER];
#endif
Vec1Tab *pTabNext;
Vec1TabList *pTabList;
};
typedef struct Vec1FlatIter Vec1FlatIter;
struct Vec1FlatIter {
/* Private variables */
const u8 *aBlob;
int nBlob;
int szRowid;
int nEntry;
int ii;
/* Public variables */
int szVec;
const u8 *aVec;
i64 iRowid;
};
typedef struct Vec1MetaValue Vec1MetaValue;
struct Vec1MetaValue {
int eType;
i64 iVal; /* Value for integer, size in bytes for T/B */
double fVal; /* Value for float type */
const u8 *pPtr; /* Pointer to data */
};
typedef struct Vec1Filter Vec1Filter;
struct Vec1Filter {
char vop; /* VEC1_OP_XXX value */
int iMeta; /* Index of meta-value column on LHS of op */
int eType; /* Type of RHS value (SQLITE_INTEGER etc.) */
i64 iVal; /* Value for integer, size in bytes for T/B */
double fVal; /* Value for float type */
u8 *pPtr; /* Pointer to data */
};
typedef struct Vec1BucketResult Vec1BucketResult;
struct Vec1BucketResult {
int iBucket;
float fDist;
};
typedef struct Vec1Query Vec1Query;
struct Vec1Query {
Vec1Tab *pTab;
/* Query parameters set by vec1SetupKANNQuery() */
i64 K; /* Number of results required */
int nProbe; /* Number of buckets to probe */
double nProbeSlack; /* Number of buckets to probe */
int bStreaming; /* True for streaming query */
int nFilter; /* Number of meta-value constraints */
Vec1Filter *aFilter; /* Array of meta-value constraints */
float *aVector; /* Vector to query NN of */
float *aTransform; /* Vector to query NN of */
u8 *aBit; /* Space for bitwise encode of 1 vector */
/* Used by streaming queries only */
int nBucket; /* Size of aBucket array */
Vec1BucketResult *aBucket; /* Remaining buckets for streaming queries */
int nOrigRes; /* Number of original results */
Vec1AnnHeap heap; /* Heap object if this is indexed query */
};
/*
** Cursor object.
**
** bSqlAnnQuery:
** Set to true if this is an NN query against an index=none table - one
** implemented by a single SQL statement.
*/
struct Vec1Csr {
sqlite3_vtab_cursor base;
sqlite3_stmt *pStmt;
/* Values used for ANN queries against %_idx data */
int iCurrentRes;
int nCurrentRes;
int bSeek; /* pStmt points at current entry already */
int bLoadConfig;
/* Cache of "distance" column for non-NN queries */
char *zDistance;
int bSqlNNQuery;
Vec1Query *pQuery;
};
#define VEC1_OP_EQ 'A'
#define VEC1_OP_LT 'B'
#define VEC1_OP_GT 'C'
#define VEC1_OP_LE 'D'
#define VEC1_OP_GE 'E'
#define VEC1_OP_IS 'F'
#define VEC1_OP_ISNULL 'G'
#define VEC1_OP_NOTNULL 'H'
#define VEC1_OP_IN 'I'
#define VEC1_OP_LIMIT 'L'
#define VEC1_OP_PARAMS 'P'
#define VEC1_QUERY_DEFAULT_NPROBE 0.05
static char *vec1Strdup(int *pRc, const char *z){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
zRet = sqlite3_mprintf("%s", z);
if( zRet==0 ){
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
static char *vec1MPrintf(int *pRc, const char *zFmt, ...){
char *zRet = 0;
va_list ap;
va_start(ap, zFmt);
zRet = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( *pRc==SQLITE_OK ){
if( zRet==0 ){
*pRc = SQLITE_NOMEM;
}
}else{
sqlite3_free(zRet);
zRet = 0;
}
return zRet;
}
static void vec1VtabError(Vec1Tab *pTab, const char *zFmt, ...){
char *zMsg = 0;
va_list ap;
va_start(ap, zFmt);
zMsg = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
sqlite3_free(pTab->base.zErrMsg);
pTab->base.zErrMsg = zMsg;
}
/*
** Prepare the SQL statement passed as the second argument. If successful,
** set (*pStmt) to point to the prepared statement handle and return
** SQLITE_OK. It is the responsibility of the caller to eventually finalize
** the statement using sqlite3_finalize().
**
** Or, if an error occurs, set (*ppStmt) to NULL, return an SQLite error
** code and leave an error message in the virtual table object.
*/
static int vec1PrepareSql(
Vec1Tab *pTab,
sqlite3_stmt **ppStmt,
const char *zFmt,
...
){
char *zSql = 0;
int rc = SQLITE_OK;
va_list ap;
va_start(ap, zFmt);
zSql = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
*ppStmt = 0;
if( zSql ){
rc = sqlite3_prepare_v2(pTab->db, zSql, -1, ppStmt, 0);
sqlite3_free(zSql);
if( rc!=SQLITE_OK ){
vec1VtabError(pTab, "%s", sqlite3_errmsg(pTab->db));
}
}else{
rc = SQLITE_NOMEM;
}
return rc;
}
#define VEC1_SQL_SCAN_BASE 1
#define VEC1_SQL_INSERT_IDX 2
#define VEC1_SQL_SCAN_IDX 3
#define VEC1_SQL_CNT_IDX 4
#define VEC1_SQL_FILTER_IDX 5
#define VEC1_SQL_WRITE_CONFIG 5
#define VEC1_SQL_INCR_CONFIG 6
#define VEC1_SQL_LOAD_CONFIG 7
static int vec1GetSql(Vec1Tab *pTab, int eSql, sqlite3_stmt **ppStmt){
const char *azSql[] = {
#define VEC1_SQL_INSERT_BASE 0
"INSERT INTO %Q.'%q_base' VALUES(?, ?%s)",
"SELECT * FROM %Q.'%q_base' ORDER BY 1",
"INSERT INTO %Q.'%q_idx'(id, bucket, first, last, val)VALUES(?,?,?,?,?)",
"SELECT val, bucket, rowid FROM %Q.'%q_idx'",
"SELECT sum( length(val) / ? ) FROM %Q.'%q_idx'",
"REPLACE INTO %Q.'%q_config'(id, val) VALUES(?, ?)",
"INSERT INTO %Q.'%q_config'(id, val) VALUES(?, 1) "
"ON CONFLICT(id) DO UPDATE SET val=val+1",
"SELECT id, val FROM %Q.'%q_config'",
#define VEC1_SQL_LOOKUP_BASE 8
"SELECT * FROM %Q.'%q_base' WHERE id=?",
#define VEC1_SQL_EDIT_IDX 9
"SELECT id, min( length(val) ) FROM %Q.'%q_idx' WHERE bucket=?",
#define VEC1_SQL_DEL_LOOKUP_BASE 10
/* Used when removing a row by id. This statement deletes the %_base
** entry and returns the vector so that the user may use it to
** find the %_idx entry to delete. */
"DELETE FROM %Q.'%q_base' WHERE id=? RETURNING vector",
#define VEC1_SQL_SEARCH_IDX 11
"SELECT id FROM %Q.'%q_idx' WHERE bucket=? AND ? BETWEEN first AND last",
#define VEC1_SQL_SEARCH_IDX_FOR_INSERT 12
"SELECT id, first, last, val "
"FROM %Q.'%q_idx' WHERE bucket=? AND length(val)<? LIMIT 1",
#define VEC1_SCAN_BUCKET 13
"SELECT val, bucket, rowid FROM %Q.'%q_idx' WHERE bucket = 0+?",
#define VEC1_SQL_UPDATE_BASE 14
"UPDATE %Q.'%q_base' SET vector = ? WHERE id=?",
#define VEC1_SQL_ZERO_BASE_RANGE 15
"UPDATE %Q.'%q_base' SET vector = 0, id=id WHERE id BETWEEN ? AND ?",
#define VEC1_SQL_DELETE_FROM_IDX 16
"DELETE FROM %Q.'%q_idx' WHERE id = ?",
#define VEC1_SQL_UPDATE_BASE2 17
"UPDATE %Q.'%q_base' SET vector = ?, id=id WHERE id=?",
#define VEC1_SQL_READ_META 18
"SELECT val FROM %Q.'%q_meta' WHERE id=?",
#define VEC1_SQL_WRITE_META 19
"REPLACE INTO %Q.'%q_meta' VALUES(?, ?)",
#define VEC1_SQL_DELETE_FROM_META 20
"DELETE FROM %Q.'%q_meta' WHERE id BETWEEN (?1<<8) AND (?1<<8)+255",
};
int rc = SQLITE_OK;
assert( eSql>=0 && eSql<size_of_array(azSql) );
assert( size_of_array(azSql)==size_of_array(pTab->aStmt) );
if( pTab->aStmt[eSql]==0 ){
if( eSql==VEC1_SQL_INSERT_BASE ){
int ii;
char *zExtra = 0;
for(ii=0; ii<pTab->nMeta; ii++){
zExtra = vec1MPrintf(&rc, "%z,?", zExtra);
}
if( rc==SQLITE_OK ){
rc = vec1PrepareSql(
pTab, &pTab->aStmt[eSql], azSql[eSql],
pTab->zDb, pTab->zName, zExtra
);
}
sqlite3_free(zExtra);
}else{
rc = vec1PrepareSql(
pTab, &pTab->aStmt[eSql], azSql[eSql], pTab->zDb, pTab->zName
);
}
}
*ppStmt = pTab->aStmt[eSql];
return rc;
}
static void vec1UnloadModel(Vec1Tab *pTab){
sqlite3_free(pTab->pModBlob);
sqlite3_free(pTab->mod.aModelT);
vec1ModelFreeTCent(&pTab->mod);
pTab->pModBlob = 0;
pTab->cfg.iModelVersion = 0;
memset(&pTab->mod, 0, sizeof(pTab->mod));
}
static void vec1UnloadConfig(Vec1Tab *pTab){
pTab->cfg.nElem = 0;
}
/*
** xDisconnect method. Free the virtual table object passed as the only
** argument.
*/
static int vec1DisconnectMethod(sqlite3_vtab *pVtab){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
int ii;
/* Remove this table object from the database handle list of vec1 tables */
if( pTab->pTabList ){
Vec1Tab **pp = &pTab->pTabList->pFirst;
while( *pp!=pTab ) pp = &(*pp)->pTabNext;
*pp = pTab->pTabNext;
}
for(ii=0; ii<size_of_array(pTab->aStmt); ii++){
sqlite3_finalize(pTab->aStmt[ii]);
}
vec1UnloadModel(pTab);
sqlite3_free(pTab->zDb);
sqlite3_free(pTab->zName);
sqlite3_free(pTab->zTrainTbl);
sqlite3_free(pTab->zIdxTbl);
sqlite3_free(pTab->aTmpVec);
sqlite3_free(pTab);
return SQLITE_OK;
}
static int vec1SqlExec(Vec1Tab *pTab, const char *zSqlfmt, char **pzErr){
char *zSql = 0;
int rc = SQLITE_OK;
zSql = sqlite3_mprintf(zSqlfmt, pTab->zDb, pTab->zName);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
sqlite3_free(zSql);
}
return rc;
}
static void vec1StmtReset(int *pRc, sqlite3_stmt *pStmt){
int rc = sqlite3_reset(pStmt);
if( *pRc==SQLITE_OK ) *pRc = rc;
}
static void vec1StmtFinalize(int *pRc, sqlite3_stmt *pStmt){
int rc = sqlite3_finalize(pStmt);
if( *pRc==SQLITE_OK ) *pRc = rc;
}
/*
** Return true if character c is an ASCII range whitespace character.
*/
static int vec1IsWhitespace(char c){
return (c==' ' || c=='\t' || c=='\r' || c=='\n');
}
/*
** Check that zName is a legal name for a vec1 column. A legal column name
** is any single token or quoted string, with no trailing garbage. For
** the purposes of this function, a token is a contiguous sequence of
** non-whitespace characters that do not begin with a quote character.
**
** If zName is a legal name (or an illegal name that will be detected by
** sqlite3_declare_vtab()), SQLITE_OK is returned and (*pzErr) is left
** unchanged. Or, if it is not a legal name, SQLITE_ERROR is returned, and
** *pzErr is left to point to a buffer containing an English language
** error message. It is the responsibility of the caller to eventually
** free this buffer using sqlite3_free().
*/
static int vec1CheckColumnName(const char *zName, char **pzErr){
char aName[16];
int nName = 0;
const char *z = zName;
while( vec1IsWhitespace(*z) ) z++;
if( (*z)=='"' || (*z)=='\'' || (*z)=='`' ){
char q = *z++;
while( 1 ){
if( (*z)=='\0' ){
*pzErr = sqlite3_mprintf("vec1: unmatched quote character");
return SQLITE_ERROR;
}
if( (*z)==q ){
z++;
if( (*z)!=q ) break;
}
if( nName<(int)sizeof(aName) ) aName[nName++] = *z;
z++;
}
}
else if( (*z)=='[' ){
z++;
while( (*z) && (*z)!=']' ){
if( nName<(int)sizeof(aName) ) aName[nName++] = *z;
z++;
}
if( (*z)=='\0' ){
*pzErr = sqlite3_mprintf("vec1: unmatched [ character");
return SQLITE_ERROR;
}
z++;
}
else {
while( (*z) && 0==vec1IsWhitespace(*z) ){
if( nName<(int)sizeof(aName) ) aName[nName++] = *z;
z++;
}
}
while( vec1IsWhitespace(*z) ) z++;
if( (*z)!='\0' ){
*pzErr = sqlite3_mprintf("vec1: trailing characters after column name");
return SQLITE_ERROR;
}
if( nName==5 && 0==sqlite3_strnicmp(aName, "rowid", 5)
|| nName==8 && 0==sqlite3_strnicmp(aName, "distance", 8)
){
*pzErr = sqlite3_mprintf("vec1: reserved column name: %s", zName);
return SQLITE_ERROR;
}
return SQLITE_OK;
}
/*
** Invoke sqlite3_declare_vtab() for this table.
*/
static int vec1DeclareVtab(
sqlite3 *db,
int argc,
const char *const* argv,
char **pzErr
){
const char *zVector = "vector";
char *zExtra = 0;
char *zCreate = 0;
int rc = SQLITE_OK;
int ii;
for(ii=3; rc==SQLITE_OK && ii<argc; ii++){
rc = vec1CheckColumnName(argv[ii], pzErr);
}
if( rc==SQLITE_OK ){
if( argc>3 ){
zVector = argv[3];
}
for(ii=4; ii<argc; ii++){
zExtra = vec1MPrintf(&rc, "%z, %s", zExtra, argv[ii]);
}
}
zCreate = vec1MPrintf(&rc,
"CREATE TABLE v1(cmd HIDDEN, arg HIDDEN, distance HIDDEN, %s BLOB%z)",
zVector, zExtra
);
if( rc==SQLITE_OK ){
rc = sqlite3_declare_vtab(db, zCreate);
sqlite3_free(zCreate);
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("vec1: %s", sqlite3_errmsg(db));
}
}
return rc;
}
/*
** Arguments:
**
** argv[0] - module name ("vec1")
** argv[1] - database name
** argv[2] - table name
*/
static int vec1CreateConnect(
int bCreate,
Vec1TabList *pTabList,
sqlite3 *db,
int argc, const char *const *argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
Vec1Tab *pTab;
int rc = SQLITE_OK;
assert( *ppVtab==0 );
sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS, 1);
pTab = sqlite3_malloc(sizeof(*pTab));
if( pTab==0 ){
return SQLITE_NOMEM;
}
memset(pTab, 0, sizeof(*pTab));
pTab->zDb = vec1Strdup(&rc, argv[1]);
pTab->zName = vec1Strdup(&rc, argv[2]);
pTab->zTrainTbl = vec1MPrintf(&rc, "%s_model", argv[2]);
pTab->zIdxTbl = vec1MPrintf(&rc, "%s_idx", argv[2]);
pTab->db = db;
pTab->mod.hdr.eDistance = VEC1_DISTANCE_L2;
pTab->nMeta = MAX(argc-4, 0);
if( pTab->nMeta>VEC1_MAX_META_COLUMNS ){
*pzErr = sqlite3_mprintf(
"too many meta-data columns (max is %d)", VEC1_MAX_META_COLUMNS
);
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK ){
rc = vec1DeclareVtab(db, argc, argv, pzErr);
}
if( rc==SQLITE_OK && bCreate ){
char *zShadowSchema = vec1ShadowSchema(pTab->zDb, pTab->zName, pTab->nMeta);
if( zShadowSchema==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(db, zShadowSchema, 0, 0, pzErr);
sqlite3_free(zShadowSchema);
}
if( rc==SQLITE_OK ){
sqlite3_stmt *pStmt = 0;
rc = vec1GetSql(pTab, VEC1_SQL_WRITE_CONFIG, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, VEC1_CONFIG_FMTVERSION);
sqlite3_bind_int(pStmt, 2, VEC1_CURRENT_FMTVERSION);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
}
}
}
if( rc!=SQLITE_OK ){
if( pTab->base.zErrMsg ){
*pzErr = pTab->base.zErrMsg;
pTab->base.zErrMsg = 0;
}
vec1DisconnectMethod((sqlite3_vtab*)pTab);
return rc;
}
/* Add the new table to the database handle list of vec1 tables. It
** will be eventually removed by vec1DisconnectMethod(). */
pTab->pTabList = pTabList;
pTab->pTabNext = pTabList->pFirst;
pTabList->pFirst = pTab;
*ppVtab = &pTab->base;
return SQLITE_OK;
}
/*
** xCreate / xConnect
*/
static int vec1ConnectMethod(
sqlite3 *db,
void *p, /* Pointer to Vec1TabList for this db */
int argc,
const char *const *argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
return vec1CreateConnect(0, (Vec1TabList*)p, db, argc, argv, ppVtab, pzErr);
}
static int vec1CreateMethod(
sqlite3 *db,
void *p, /* Pointer to Vec1TabList for this db */
int argc,
const char *const *argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
return vec1CreateConnect(1, (Vec1TabList*)p, db, argc, argv, ppVtab, pzErr);
}
static int vec1DestroyMethod(sqlite3_vtab *pVtab){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
int rc = SQLITE_OK;
char *zDrop = vec1DropShadowSchema(pTab->zDb, pTab->zName);
if( zDrop==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(pTab->db, zDrop, 0, 0, 0);
sqlite3_free(zDrop);
}
if( rc==SQLITE_OK ){
vec1DisconnectMethod(pVtab);
}
return rc;
}
/*
** Cursor methods
*/
static int vec1Open(
sqlite3_vtab *pVtab,
sqlite3_vtab_cursor **ppCursor
){
Vec1Csr *pCur;
UNUSED_PARAMETER(pVtab);
pCur = sqlite3_malloc(sizeof(*pCur));
if( pCur==0 ){
return SQLITE_NOMEM;
}
memset(pCur, 0, sizeof(*pCur));
*ppCursor = &pCur->base;
return SQLITE_OK;
}
static void vec1QueryFree(Vec1Query *pQuery){
if( pQuery ){
int ii;
for(ii=0; ii<pQuery->nFilter; ii++){
sqlite3_free(pQuery->aFilter[ii].pPtr);
}
sqlite3_free(pQuery->heap.aRes);
sqlite3_free(pQuery->aBucket);
sqlite3_free(pQuery);
}
}
static void vec1ResetCsr(Vec1Csr *pCsr){
vec1QueryFree(pCsr->pQuery);
sqlite3_finalize(pCsr->pStmt);
pCsr->pStmt = 0;
pCsr->bSeek = 0;
sqlite3_free(pCsr->zDistance);
pCsr->zDistance = 0;
pCsr->bSqlNNQuery = 0;
}
static int vec1Close(sqlite3_vtab_cursor *cur){
Vec1Csr *pCsr = (Vec1Csr*)cur;
vec1ResetCsr(pCsr);
sqlite3_free(pCsr);
return SQLITE_OK;
}
static void vec1FlatIterNext(Vec1FlatIter *pIter){
while( 1 ){
pIter->ii++;
if( pIter->ii>=pIter->nEntry ){
pIter->iRowid = 0;
pIter->aVec = 0;
break;
}else{
i64 iRowid = 0;
int iOff = VEC1_LIST_SZHDR + pIter->ii * pIter->szRowid;
if( pIter->szRowid==4 ){
iRowid = (i64)vec1GetU32(&pIter->aBlob[iOff]);
if( iRowid==VEC1_TOMBSTONE_32 ) continue;
}else{
iRowid = (i64)vec1GetU64(&pIter->aBlob[iOff]);
if( iRowid==VEC1_TOMBSTONE_64 ) continue;
}
iOff = VEC1_LIST_SZHDR +
(pIter->nEntry * pIter->szRowid) +
(pIter->ii * pIter->szVec);
pIter->aVec = &pIter->aBlob[iOff];
pIter->iRowid = iRowid;
break;
}
}
}
/*
** Return the encoded vector size for the model. Two cases:
**
** a) PQ encoding. In this case encoded vector size is hdr.nCodebook.
**
** b) Bitwise encoding. In this case encoded vector size is 4 bytes +
** 1 bit per dimension.
*/
static int vec1EncodedVectorSize(const Vec1Model *pMod){
int ret = (int)pMod->hdr.nCodebook;
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ) ret += sizeof_f32;
return ret;
}
/*
** nBlob byte buffer aBlob[] contains a list read from the %_idx table.
** This function returns zero if the list size matches the embedded
** number-of-entries field, and so the list may be accessed safely without
** bounds checking, or non-zero otherwise.
*/
static int vec1CheckIdxSize(Vec1Tab *p, const u8 *aBlob, i64 nBlob){
int bRet = 1;
/* If the blob is smaller than 8 bytes, it must be corrupt */
if( nBlob>=VEC1_LIST_SZHDR ){
/* Read the flags and number-of-entries fields */
u32 flags = vec1GetU32(&aBlob[0]);
i64 nEntry = (int)vec1GetU32(&aBlob[4]);
const i64 szRowid = ((flags & VEC1_LIST_64BIT) ? 8 : 4);
if( p->mod.hdr.nCodebook>0 ){
const i64 nBlk = ((nEntry+VEC1_PQ_BLOCKSIZE-1) / VEC1_PQ_BLOCKSIZE);
const i64 szBlk = VEC1_PQ_BLOCKSIZE * vec1EncodedVectorSize(&p->mod);
bRet = (nBlob!=(VEC1_LIST_SZHDR + nEntry*szRowid + nBlk*szBlk));
}else{
const i64 szVec = p->cfg.nElem*sizeof_f32;
bRet = (nBlob!=(VEC1_LIST_SZHDR + nEntry*szRowid + nEntry*szVec));
}
}
return bRet;
}
static int vec1FlatIterStart(
Vec1Tab *pTab,
Vec1FlatIter *pIter,
const u8 *aBlob,
int nBlob
){
u32 flags;
pIter->aBlob = aBlob;
pIter->nBlob = nBlob;
pIter->ii = 0;
if( vec1CheckIdxSize(pTab, aBlob, nBlob) ){
return VEC1_CORRUPT;
}
flags = vec1GetU32(aBlob);
pIter->szRowid = (flags & VEC1_LIST_64BIT) ? 8 : 4;
pIter->nEntry = vec1GetU32(&aBlob[4]);
pIter->szVec = (pTab->cfg.nElem * sizeof_f32);
pIter->ii = -1;
vec1FlatIterNext(pIter);
return SQLITE_OK;
}
/*
** Merge the two sorted subarrays a[p..q-1] and a[q..r-1] using
** the temporary buffer aTmp.
*/
static void vec1Merge(
Vec1AnnResult *a,
Vec1AnnResult *aTmp,
int p,
int q,
int r
){
int i = p;
int j = q;
int k = p;
while( i<q && j<r ){
if( a[i].fDist <= a[j].fDist ){
aTmp[k++] = a[i++];
}else{
aTmp[k++] = a[j++];
}
}
while( i<q ){
aTmp[k++] = a[i++];
}
while( j<r ){
aTmp[k++] = a[j++];
}
for(i=p; i<r; i++){
a[i] = aTmp[i];
}
}
/*
** Recursive merge-sort implementation.
*/
static void vec1MergeSort(
Vec1AnnResult *a,
Vec1AnnResult *aTmp,
int p,
int r
){
int q;
if( r-p<=1 ) return;
q = p + (r-p)/2;
vec1MergeSort(a, aTmp, p, q);
vec1MergeSort(a, aTmp, q, r);
vec1Merge(a, aTmp, p, q, r);
}
/*
** Sort array a[0..n-1] in ascending order of fDist.
*/
static int vec1AnnResultSort(Vec1AnnResult *a, i64 n){
int rc = SQLITE_OK;
if( n>1 ){
i64 nByte = sizeof(Vec1AnnResult) * n;
Vec1AnnResult *aTmp = (Vec1AnnResult*)sqlite3_malloc64(nByte);
if( aTmp==0 ){
rc = SQLITE_NOMEM;
}else{
vec1MergeSort(a, aTmp, 0, (int)n);
sqlite3_free(aTmp);
}
}
return rc;
}
static int vec1DoKANNBucket(Vec1Query*,int,Vec1Buffer*,Vec1Buffer*,Vec1Buffer*);
static int vec1StreamingNext(Vec1Csr *pCsr){
int rc = SQLITE_OK;
Vec1Query *pQuery = pCsr->pQuery;
Vec1AnnHeap *pHeap = &pQuery->heap;
int ii;
int nSort = (int)(pHeap->nRes - pCsr->nCurrentRes);
{
i64 nRes = pHeap->nRes;
pHeap->nRes = 0;
for(ii=pCsr->nCurrentRes; ii<nRes; ii++){
vec1HeapInsert(pHeap, pHeap->aRes[ii].iRowid, pHeap->aRes[ii].fDist);
}
}
while( pQuery->nBucket>0 && nSort<=(pQuery->nOrigRes/2) ){
Vec1Buffer a = {0,0,0};
Vec1Buffer b = {0,0,0};
Vec1Buffer c = {0,0,0};
int iBest = 0;
int iBucket = 0;
/* Find the nearest remaining bucket to the query vector. Remove it
** from the aBucket[] array. */
for(ii=1; ii<pQuery->nBucket; ii++){
if( pQuery->aBucket[ii].fDist<pQuery->aBucket[iBest].fDist ){
iBest = ii;
}
}
iBucket = pQuery->aBucket[iBest].iBucket;
pQuery->nBucket--;
if( iBest!=pQuery->nBucket ){
pQuery->aBucket[iBest] = pQuery->aBucket[pQuery->nBucket];
}
rc = vec1DoKANNBucket(pQuery, iBucket, &a, &b, &c);
vec1BufferFree(&a);
vec1BufferFree(&b);
vec1BufferFree(&c);
if( rc!=SQLITE_OK ) return rc;
}
nSort = (int)MIN(pHeap->nMax, pHeap->nRes);
rc = vec1AnnResultSort(pHeap->aRes, nSort);
if( rc==SQLITE_OK ){
pCsr->iCurrentRes = 0;
pCsr->nCurrentRes = nSort;
}
return rc;
}
/*
** xNext() method for vec1 cursors.
*/
static int vec1NextMethod(sqlite3_vtab_cursor *cur){
int rc = SQLITE_OK;
Vec1Csr *pCsr = (Vec1Csr *)cur;
if( pCsr->zDistance ){
sqlite3_free(pCsr->zDistance );
pCsr->zDistance = 0;
}
if( pCsr->pQuery ){
pCsr->iCurrentRes++;
pCsr->bSeek = 0;
if( pCsr->iCurrentRes>=pCsr->nCurrentRes ){
/* If this is a streaming query, try to get some more results. */
if( pCsr->pQuery->bStreaming ){
rc = vec1StreamingNext(pCsr);
}
/* If this was not a streaming query, or no further results are
** available, set pStmt to 0 to indicate EOF. */
if( pCsr->iCurrentRes>=pCsr->nCurrentRes ){
sqlite3_finalize(pCsr->pStmt);
pCsr->pStmt = 0;
}
}
sqlite3_reset(pCsr->pStmt);
}else{
rc = sqlite3_step(pCsr->pStmt);
if( rc!=SQLITE_ROW ){
rc = sqlite3_finalize(pCsr->pStmt);
pCsr->pStmt = 0;
}else{
rc = SQLITE_OK;
}
}
return rc;
}
static int vec1ModelTransform(
Vec1Model *pMod,
float **paOut
){
if( pMod->aModel ){
const int nCodebook = pMod->hdr.nCodebook;
const int nCodeElem = pMod->nCodeElem;
int M;
float *pOut = (float*)sqlite3_malloc(
sizeof_f32 * nCodebook * nCodeElem * VEC1_PQ_CODEBOOK_SZ
);
if( pOut==0 ) return SQLITE_NOMEM;
*paOut = pOut;
/* For VEC1_TRANSPOSE_WIDTH==8, the order of elements in output is:
**
** aModel[M=0][d=0] [K=0..7]
** aModel[M=0][d=1] [K=0..7]
** aModel[M=0][d=2] [K=0..7]
** ...
** aModel[M=0][d=(nCodeElem-1)][K=0..7]
** aModel[M=0][d=0] [K=8..15]
** aModel[M=0][d=1] [K=8..15]
** aModel[M=0][d=2] [K=8..15]
** ...
** aModel[M=0][d=(nCodeElem-1)][K=248..255]
** aModel[M=1][d=0] [K=0..7]
** ...
*/
for(M=0; M<nCodebook; M++){
vec1CodebookTranspose(
&pMod->aModel[ M * nCodeElem * VEC1_PQ_CODEBOOK_SZ ],
nCodeElem, VEC1_TRANSPOSE_WIDTH, VEC1_PQ_CODEBOOK_SZ,
&pOut[ M * nCodeElem * VEC1_PQ_CODEBOOK_SZ ]
);
}
}
return SQLITE_OK;
}
/*
** If it is not already loaded, load the PQ model from disk.
*/
static int vec1LoadModel(Vec1Tab *pTab){
int rc = SQLITE_OK;
sqlite3 *db = pTab->db;
sqlite3_blob *pBlob = 0;
vec1UnloadModel(pTab);
rc = sqlite3_blob_open(db, pTab->zDb, pTab->zTrainTbl, "val", 1, 0, &pBlob);
if( rc==SQLITE_OK ){
int nByte = sqlite3_blob_bytes(pBlob);
void *pData = sqlite3_malloc(nByte);
if( pData==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_blob_read(pBlob, pData, nByte, 0);
}
if( rc==SQLITE_OK ){
rc = vec1DecodeModel(pData, nByte, &pTab->mod, &pTab->base.zErrMsg);
}
if( rc==SQLITE_OK ){
pTab->pModBlob = pData;
}else{
sqlite3_free(pData);
}
sqlite3_blob_close(pBlob);
}
if( rc==SQLITE_OK ){
rc = vec1ModelTransform(&pTab->mod, &pTab->mod.aModelT);
}
return rc;
}
#if 0
static int vec1AnnResultPartition(Vec1AnnResult *aArr, int iLeft, int iRight){
double fPivot = aArr[iRight].fDist; /* choose last element */
int iSmall = iLeft; /* Next small element goes here */
int ii;
for(ii=iLeft; ii<iRight; ii++){
if( aArr[ii].fDist<fPivot ){
SWAP(Vec1AnnResult, aArr[iSmall], aArr[ii]);
iSmall++;
}
}
SWAP(Vec1AnnResult, aArr[iSmall], aArr[iRight]);
return iSmall;
}
static void vec1AnnResultSelect(Vec1AnnResult *aArr, int nArr, int K){
int iLeft = 0;
int iRight = nArr-1;
while (iLeft <= iRight) {
size_t iPivot = vec1AnnResultPartition(aArr, iLeft, iRight);
if( iPivot==K ){
return;
}else if( iPivot>K ){
iRight = iPivot - 1;
} else {
iLeft = iPivot + 1;
}
}
}
#endif
/*
** Merge the two sorted subarrays a[p..q-1] and a[q..r-1] using
** the temporary buffer aTmp.
*/
static void vec1BucketMerge(
Vec1BucketResult *a,
Vec1BucketResult *aTmp,
int p,
int q,
int r
){
int i = p;
int j = q;
int k = p;
while( i<q && j<r ){
if( a[i].fDist <= a[j].fDist ){
aTmp[k++] = a[i++];
}else{
aTmp[k++] = a[j++];
}
}
while( i<q ){
aTmp[k++] = a[i++];
}
while( j<r ){
aTmp[k++] = a[j++];
}
for(i=p; i<r; i++){
a[i] = aTmp[i];
}
}
/*
** Recursive merge-sort implementation.
*/
static void vec1BucketMergeSort(
Vec1BucketResult *a,
Vec1BucketResult *aTmp,
int p,
int r
){
int q;
if( r-p<=1 ) return;
q = p + (r-p)/2;
vec1BucketMergeSort(a, aTmp, p, q);
vec1BucketMergeSort(a, aTmp, q, r);
vec1BucketMerge(a, aTmp, p, q, r);
}
/*
** Sort array a[0..n-1] in ascending order of fDist.
*/
static int vec1BucketSort(Vec1BucketResult *a, i64 n){
int rc = SQLITE_OK;
if( n>1 ){
i64 nByte = sizeof(Vec1BucketResult) * n;
Vec1BucketResult *aTmp = (Vec1BucketResult*)sqlite3_malloc64(nByte);
if( aTmp==0 ){
rc = SQLITE_NOMEM;
}else{
vec1BucketMergeSort(a, aTmp, 0, (int)n);
sqlite3_free(aTmp);
}
}
return rc;
}
/*
** Partition the part of the array between elements iLeft and iRight
** (inclusive). Return the index of the partition key.
*/
static int vec1BucketPartition(Vec1BucketResult *aArr, int iLeft, int iRight){
float fPivot = aArr[iRight].fDist; /* choose last element */
int iSmall = iLeft; /* Next small element goes here */
int ii;
for(ii=iLeft; ii<iRight; ii++){
if( aArr[ii].fDist<fPivot ){
SWAP(Vec1BucketResult, aArr[iSmall], aArr[ii]);
iSmall++;
}
}
SWAP(Vec1BucketResult, aArr[iSmall], aArr[iRight]);
return iSmall;
}
/*
** Use quick-select to shuffle array aArr[] so that the first K elements
** all have smaller fDist values that then remaining (nArr-K) elements.
*/
static void vec1BucketSelect(Vec1BucketResult *aArr, int nArr, int K){
int iLeft = 0;
int iRight = nArr-1;
assert( iLeft<=iRight && K<=iRight && K>=0 );
while( 1 ){
int iPivot = vec1BucketPartition(aArr, iLeft, iRight);
if( iPivot==K ){
return;
}else if( iPivot>K ){
iRight = iPivot - 1;
} else {
iLeft = iPivot + 1;
}
}
}
static int vec1FindBuckets(
Vec1Tab *pTab,
const float *aVec,
int nProbe,
Vec1BucketResult **paBucket
){
int nBucket = pTab->mod.hdr.nBucket;
int nElem = pTab->mod.hdr.nElem;
Vec1BucketResult *aRes = 0;
int rc = SQLITE_OK;
aRes = (Vec1BucketResult*)sqlite3_malloc(sizeof(Vec1BucketResult) * nBucket);
if( aRes ){
int ii;
for(ii=0; ii<nBucket; ii++){
const float *aCentroid = &pTab->mod.aCentroid[ii*nElem];
aRes[ii].iBucket = ii;
aRes[ii].fDist = (float)vec1L2Dist(aVec, aCentroid, nElem);
}
if( nProbe<nBucket ){
vec1BucketSelect(aRes, nBucket, nProbe);
rc = vec1BucketSort(aRes, nProbe);
#ifndef NDEBUG
for(ii=0; ii<nBucket; ii++){
if( ii<nProbe ){
assert( aRes[ii].fDist<=aRes[nProbe].fDist );
}else{
assert( aRes[ii].fDist>=aRes[nProbe].fDist );
}
}
#endif
#if 0
for(ii=0; ii<nProbe; ii++){
printf("%f ", aRes[ii].fDist);
}
printf("\n");
#endif
}
}else{
rc = SQLITE_NOMEM;
}
*paBucket = aRes;
return rc;
}
static const float *vec1TransformInputVector(
const Vec1Model *pMod,
float *aTmp,
const float *aInput
){
const float *aRet = aInput;
if( (pMod->hdr.flags & VEC1_MODEL_ROTATE) ){
vec1RotateVector(pMod->hdr.nElem, pMod->aRotation, aInput, aTmp);
if( pMod->hdr.eDistance==VEC1_DISTANCE_COS ){
vec1NormalizeVector(aTmp, pMod->hdr.nElem);
}
aRet = aTmp;
}
else if( (pMod->hdr.flags & VEC1_MODEL_WHT) ){
memcpy(aTmp, aInput, pMod->hdr.nElem * sizeof_f32);
vec1ApplyWHT(aTmp, pMod->hdr.nElem);
aRet = aTmp;
}
else if( pMod->hdr.eDistance==VEC1_DISTANCE_COS ){
memcpy(aTmp, aInput, pMod->hdr.nElem * sizeof_f32);
vec1NormalizeVector(aTmp, pMod->hdr.nElem);
aRet = aTmp;
}
if( aRet!=aTmp
&& pMod->hdr.nCodebook>0
&& (pMod->hdr.nCodebook*pMod->nCodeElem)!=pMod->hdr.nElem
){
memcpy(aTmp, aRet, pMod->hdr.nElem * sizeof_f32);
aRet = aTmp;
}
return aRet;
}
/*
** Return true if vec1TransformRequired() is not a no-op. i.e. if any
** transformation is required for input vectors.
*/
static int vec1TransformRequired(const Vec1Model *pMod){
Vec1ModelHeader const *p = &pMod->hdr;
return
(p->flags & (VEC1_MODEL_ROTATE|VEC1_MODEL_WHT))
|| (p->eDistance==VEC1_DISTANCE_COS);
}
/*
** This routine is the core of both the LUT builder and the vector encoder.
**
** pIn:
** pIn is an array of VEC1_TRANSPOSE_WIDTH nCodeElem-dimension vectors in
** column major format. That is to say, pIn[0] is the first element of
** the first vector, pIn[1] is the first element of the second vector,
** and so on.
**
** In other words, to find element i of vector v:
**
** pIn[i * VEC1_TRANSPOSE_WIDTH + v]
**
** aVec:
** A single vector with nCodeElem elements..
**
** nCodeElem:
** Size of vectors in elements.
**
** aOut:
** Array of VEC1_TRANSPOSE_WIDTH. Populated with the square of the L2 distance
** between aVec[] and each of the vectors in pIn.
*/
static void vec1ModelTDist(
const float *pIn,
const float *aVec,
int nCodeElem,
float *aOut
){
int d;
#if defined(VEC1_HAVE_AVX2)
__m256 acc[ VEC1_TRANSPOSE_WIDTH / 8 ];
const int nAcc = (VEC1_TRANSPOSE_WIDTH / 8);
int c;
for(c=0; c<nAcc; c++) acc[c] = _mm256_setzero_ps();
for(d=0; d<nCodeElem; d++){
__m256 v = _mm256_set1_ps(aVec[d]);
for(c=0; c<nAcc; c++){
__m256 diff = _mm256_sub_ps(v, LOADU(&pIn[c*8]));
acc[c] = FMADD(diff, diff, acc[c]);
}
pIn += VEC1_TRANSPOSE_WIDTH;
}
for(c=0; c<nAcc; c++) _mm256_storeu_ps(&aOut[c*8], acc[c]);
#elif defined(VEC1_HAVE_NEON)
float32x4_t acc[ VEC1_TRANSPOSE_WIDTH / 4 ];
const int nAcc = (VEC1_TRANSPOSE_WIDTH / 4);
int c;
for(c=0; c<nAcc; c++) acc[c] = vdupq_n_f32(0.0f);
for(d=0; d<nCodeElem; d++){
float32x4_t v = vdupq_n_f32( aVec[d] );
for(c=0; c<nAcc; c++){
float32x4_t diff = vsubq_f32(v, vld1q_f32(&pIn[c*4]));
acc[c] = vmlaq_f32(acc[c], diff, diff);
}
pIn += VEC1_TRANSPOSE_WIDTH;
}
for(c=0; c<nAcc; c++) vst1q_f32(&aOut[c*4], acc[c]);
#else
memset(aOut, 0, sizeof_f32 * VEC1_TRANSPOSE_WIDTH);
for(d=0; d<nCodeElem; d++){
float vv = aVec[d];
const float *aVC = pIn;
int ii;
for(ii=0; ii<VEC1_TRANSPOSE_WIDTH; ii++){
aOut[ii] += ((vv - aVC[ii]) * (vv - aVC[ii]));
}
pIn += VEC1_TRANSPOSE_WIDTH;
}
#endif
}
/*
** Write value v into buffer aBuf[] as an SQLite varint.
*/
static int vec1PutVarint(u8 *aBuf, u64 v){
if( v<=0x7f ){
aBuf[0] = v&0x7f;
return 1;
}
if( v<=0x3fff ){
aBuf[0] = ((v>>7)&0x7f)|0x80;
aBuf[1] = v&0x7f;
return 2;
}
{
u8 a[10];
int ii;
int nRet;
for(nRet=0; v; nRet++){
a[nRet] = (v&0x7F);
v = v >> 7;
}
for(ii=0; ii<nRet; ii++){
aBuf[ii] = a[nRet-1-ii] | ((ii==nRet-1) ? 0x00 : 0x80);
}
return nRet;
}
}
static int vec1GetVarint(const u8 *aBuf, u64 *piVal){
u64 out = 0;
int ii = 0;
out = (aBuf[0] & 0x7F);
if( aBuf[0] & 0x80 ){
for(ii=1; ii<9; ii++){
out = (out<<7) + (aBuf[ii] & 0x7F);
if( (aBuf[ii] & 0x80)==0 ){
break;
}
}
}
*piVal = out;
return ii+1;
}
/*
** Populate an LUT to use to calculate distances when scanning a list of
** PQ codes. Parameter aVec[] is the (possibly residual) vector that will be
** compared with the PQ encoded vectors.
*/
static int vec1PQLUTBuild(
Vec1Tab *pTab,
const float *aVec, /* Residual vector */
float *aDist /* Populate this nCodebook*256 array */
){
const int nCodebook = pTab->mod.hdr.nCodebook;
const int nCodeElem = pTab->mod.nCodeElem;
int M, K;
float *pIn = pTab->mod.aModelT;
for(M=0; M<nCodebook; M++){
for(K=0; K<VEC1_PQ_CODEBOOK_SZ; K+=VEC1_TRANSPOSE_WIDTH){
vec1ModelTDist(
pIn, &aVec[M*nCodeElem], nCodeElem,
&aDist[(M * VEC1_PQ_CODEBOOK_SZ) + K]
);
pIn += (VEC1_TRANSPOSE_WIDTH * nCodeElem);
}
}
return SQLITE_OK;
}
/*
** Size of the LUT for a scan of BQ compressed nElem element vectors.
*/
static int vec1BinaryQuantSize(int nElem){
const int B = 8;
return (((nElem+(B-1))/B) * (1<<B) * sizeof_f32);
}
static int vec1PQLUTSize(int nCodebook){
return (nCodebook * VEC1_PQ_CODEBOOK_SZ) * sizeof_f32;
}
static void vec1BinaryQuantLUTBuild(
Vec1Tab *pTab,
const float *aVec, /* Residual vector */
float *aDist /* Populate this nCodebook*256 array */
){
int nVec = pTab->cfg.nElem;
int nCodebook = pTab->mod.hdr.nCodebook;
const int nCodeElem = pTab->mod.nCodeElem;
int M;
float c = 1.0f / sqrtf((float)nVec);
assert( nCodeElem==8 || nCodeElem==16 );
for(M=0; M<nCodebook; M++){
float aZero[8];
float *aLUT = &aDist[M*256];
const float *aSub = &aVec[M*nCodeElem];
int iNext = 1;
int ii;
aLUT[0] = 0.0;
if( nCodeElem==8 ){
for(ii=0; ii<8; ii++){
assert( (ii+M*nCodeElem)<nVec || aSub[ii]==0.0 );
aZero[ii] = (aSub[ii] * c);
aLUT[0] -= aZero[ii];
}
}else{
for(ii=0; ii<16; ii+=2){
aZero[ii/2] = (aSub[ii] + aSub[ii+1]) * c;
aLUT[0] -= aZero[ii/2];
}
}
for(ii=0; ii<8; ii++){
int jj;
float add = 2 * aZero[ii];
for(jj=0; jj<iNext; jj++){
aLUT[iNext+jj] = (aLUT[jj] + add);
}
iNext = iNext+iNext;
}
}
}
/*
** Read an entry from the %_meta table into buffer pBuf.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise.
*/
static int vec1ReadMeta(
Vec1Tab *pTab, /* Vec1 virtual table */
u32 nExpect, /* Expected number of entries */
Vec1Buffer *pBuf, /* Buffer to read into */
i64 iId, /* Id of %_idx row */
int iMeta /* Index of meta-column */
){
sqlite3_stmt *pStmt = 0;
int rc = vec1GetSql(pTab, VEC1_SQL_READ_META, &pStmt);
assert( iMeta>=0 && iMeta<VEC1_MAX_META_COLUMNS );
if( rc==SQLITE_OK ){
i64 iMetaId = (iId << VEC1_META_COLUMN_BITS) + iMeta;
pBuf->n = 0;
sqlite3_bind_int64(pStmt, 1, iMetaId);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
int nByte = sqlite3_column_bytes(pStmt, 0);
rc = vec1BufferGrow(pBuf, nByte + VEC1_META_PADDING);
if( rc==SQLITE_OK ){
const u8 *a = sqlite3_column_blob(pStmt, 0);
memcpy(pBuf->a, a, nByte);
memset(&pBuf->a[nByte], 0, VEC1_META_PADDING);
pBuf->n = nByte;
}
}
vec1StmtReset(&rc, pStmt);
/* Check, so far as is possible, that the meta-value blob is the
** expected size. It is not possible to check if the meta-value blob
** uses the GENERIC format, due to the variable length fields. Code
** that handles GENERIC meta-value blobs has to dynamically verify
** the buffer size. */
if( rc==SQLITE_OK ){
if( pBuf->n<VEC1_META_SZHDR ){
rc = VEC1_CORRUPT;
}else{
u32 tflag = (vec1GetU32(&pBuf->a[0]) & VEC1_META_TYPEMASK);
u32 nEntry = vec1GetU32(&pBuf->a[4]);
i64 sz = 0;
if( nEntry!=nExpect ){
rc = VEC1_CORRUPT;
}
if( tflag==VEC1_META_1BYTEINT ){
sz = 1;
}else if( tflag==VEC1_META_4BYTEINT ){
sz = sizeof_u32;
}else if( tflag==VEC1_META_REAL ){
sz = sizeof_f64;
}else if( tflag!=VEC1_META_GENERIC ){
rc = VEC1_CORRUPT;
}
if( sz && pBuf->n!=(VEC1_META_SZHDR + sz*nEntry) ){
rc = VEC1_CORRUPT;
}
}
}
}
return rc;
}
/*
** 0: NULL value. 0 byte payload.
** 1: Integer value. 1 byte payload.
** 2: Integer value. 4 byte payload.
** 3: Integer value. 8 byte payload.
** 4: Real value. 8 byte payload.
*/
static int vec1MetaValueRead(
Vec1Buffer *pMeta,
int *piOff,
Vec1MetaValue *pVal
){
i64 iOff = (i64)*piOff;
switch( pMeta->a[iOff] ){
case 0: {
pVal->eType = SQLITE_NULL;
iOff++;
break;
}
case 1: {
pVal->eType = SQLITE_INTEGER;
pVal->iVal = (i64)(u8)pMeta->a[iOff+1];
iOff += 2;
break;
}
case 2: {
pVal->eType = SQLITE_INTEGER;
pVal->iVal = (int)vec1GetU32(&pMeta->a[iOff+1]);
iOff += 5;
break;
}
case 3: {
pVal->eType = SQLITE_INTEGER;
pVal->iVal = (i64)vec1GetU64(&pMeta->a[iOff+1]);
iOff += 9;
break;
}
case 4: {
u64 iVal = vec1GetU64(&pMeta->a[iOff+1]);
memcpy(&pVal->fVal, &iVal, sizeof(u64));
pVal->eType = SQLITE_FLOAT;
iOff += 9;
break;
}
default: {
u64 eType = 0;
iOff += vec1GetVarint(&pMeta->a[iOff], &eType);
pVal->pPtr = &pMeta->a[iOff];
pVal->iVal = (((eType-5) / 2) & (u64)0xFFFFFFFF);
iOff += pVal->iVal;
pVal->eType = (eType & 0x01) ? SQLITE_TEXT : SQLITE_BLOB;
break;
}
}
if( iOff>pMeta->n || iOff<0 ){
return VEC1_CORRUPT;
}
*piOff = (int)iOff;
return SQLITE_OK;
}
/*
** Compare numeric values a and b. Return -1, 0 or +1 if a is smaller than,
** equal to, or greater than b, respectively. i.e. (a) - (b).
*/
#define COMPARE(a, b) ((a)==(b) ? 0 : ((a)>(b) ? +1 : -1))
/*
** Return 0 if the meta value passes the filter, or 1 if it is excluded
** by the filter. This is the slow path that can do any operator for
** any types of values.
*/
static int vec1MetaValueFilter(Vec1Filter *pFilter, Vec1MetaValue *pMeta){
int cmp = 0;
if( pFilter->vop==VEC1_OP_IN ){
int ii;
for(ii=0; ii<pFilter->iVal; ii++){
if( vec1MetaValueFilter(&pFilter[ii+1], pMeta)==0 ) return 0;
}
return 1;
}
if( pMeta->eType==SQLITE_INTEGER ){
switch( pFilter->eType ){
case SQLITE_NULL: return 1;
case SQLITE_INTEGER: {
cmp = COMPARE(pFilter->iVal, pMeta->iVal);
break;
}
case SQLITE_FLOAT: {
cmp = COMPARE(pFilter->fVal, (double)pMeta->iVal);
break;
}
default: {
/* All text and blob are bigger than all numbers */
cmp = +1;
break;
}
}
}
else if( pMeta->eType==SQLITE_FLOAT ){
switch( pFilter->eType ){
case SQLITE_NULL: return 1;
case SQLITE_INTEGER: {
cmp = COMPARE((double)pFilter->iVal, pMeta->fVal);
break;
}
case SQLITE_FLOAT: {
cmp = COMPARE(pFilter->fVal, pMeta->fVal);
break;
}
default: {
/* All text and blob are bigger than all numbers */
cmp = +1;
break;
}
}
}else if( pMeta->eType==SQLITE_NULL ){
if( pFilter->vop==VEC1_OP_IS && pFilter->eType==SQLITE_NULL ) return 0;
return 1;
}else if( pMeta->eType==SQLITE_TEXT ){
switch( pFilter->eType ){
case SQLITE_NULL: return 1;
case SQLITE_INTEGER:
case SQLITE_FLOAT: {
cmp = -1;
break;
}
case SQLITE_TEXT: {
i64 n = MIN(pMeta->iVal, pFilter->iVal);
if( n>0 ){
cmp = memcmp(pFilter->pPtr, pMeta->pPtr, n);
}
if( cmp==0 ) cmp = COMPARE(pFilter->iVal, pMeta->iVal);
break;
}
default: {
assert( pFilter->eType==SQLITE_BLOB );
cmp = 1;
break;
}
}
}else{
assert( pMeta->eType==SQLITE_BLOB );
switch( pFilter->eType ){
case SQLITE_NULL: return 1;
case SQLITE_INTEGER:
case SQLITE_FLOAT:
case SQLITE_TEXT: {
cmp = -1;
break;
}
default: {
i64 n = MIN(pMeta->iVal, pFilter->iVal);
if( n>0 ) cmp = memcmp(pFilter->pPtr, pMeta->pPtr, n);
if( cmp==0 ) cmp = COMPARE(pFilter->iVal, pMeta->iVal);
assert( pFilter->eType==SQLITE_BLOB );
break;
}
}
}
switch( pFilter->vop ){
case VEC1_OP_IS:
case VEC1_OP_EQ: return !(cmp==0);
case VEC1_OP_LT: return !(cmp>0);
case VEC1_OP_LE: return !(cmp>=0);
case VEC1_OP_GT: return !(cmp<0);
case VEC1_OP_GE: return !(cmp<=0);
}
assert( pFilter->vop==VEC1_OP_NOTNULL );
return 0;
}
/*
** This is called as part of meta-data filtering when the meta-data field
** in question is stored in either VEC1_META_1BYTEINT or VEC1_META_4BYTEINT
** format.
**
** iMin, iMax, eNull:
** These three parameters describe the integer format used to store
** the meta-data field (either VEC1_META_1BYTEINT or VEC1_META_4BYTEINT).
** They pass the minimum integer value, maximum possible integer value
** and the value used to indicate NULL. So, for the two formats, they
** are:
**
** VEC1_META_1BYTEINT: 0, 254, 255.
** VEC1_META_4BYTEINT: -2147483648, +2147483646, +2147483647
**
** pOp:
** Output parameter. If it is guaranteed that no entries in the meta-data
** field will match the filter (i.e. so zero rows will be returned), set
** this to 0. Otherwise, set it to one of the VEC1_OP_XXX constants to
** indicate the operation that should be used to test each value in the
** meta-data field.
**
** pRval:
** OUTPUT parameter. If (*pOp) is VEC1_OP_IN, then this is set to the
** number of entries in the (*paRval) array output. Otherwise, it is
** set to the integer value to use as the RHS of the comparison when
** testing each meta-data value.
**
** paRval:
** OUTPUT parameter. If (*pOp) is VEC1_OP_IN, an array of values to
** compare each meta-data value to. Otherwise, 0.
**
** Return SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM)
** if an error occurs.
*/
static int vec1FilterToIntOp(
Vec1Filter *pFilter, /* The constraint doing the filtering */
i64 iMin, i64 iMax, i64 eNull, /* Meta-data format description */
int *pRval,
int **paRval,
u8 *pOp
){
/* Set the following three values as follows:
**
** res: 0 if the result can be determined to be false for all list
** entries. 1 if it is true for all non-null list entries. Or
** -1 if scanning the list using op and rval is required to
** determine which list entries match the condition.
**
** op: If res is -1, one of VEC1_OP_EQ, VEC1_OP_GT, VEC1_OP_GE,
** VEC1_OP_LT or VEC1_OP_LE.
**
** rval: A value to compare using op with all list entries.
**
** Then, if res is not 0 or 1, run through the list comparing each
** list value with value rval using operation op.
*/
int res = -1;
int rval = 0;
u8 op = 0;
assert( (iMin==0 && iMax==254 && eNull==255) || (
iMin==VEC1_META_4BYTEMIN
&& iMax==VEC1_META_4BYTEMAX
&& eNull==VEC1_META_4BYTENULL
));
if( pFilter->vop==VEC1_OP_IN ){
int *aRval = (int*)vec1MallocZero(sizeof_u32 * (pFilter->iVal+1));
if( aRval==0 ){
return SQLITE_NOMEM;
}else{
int ii;
int nRval = 0;
for(ii=0; ii<pFilter->iVal; ii++){
vec1FilterToIntOp(&pFilter[ii+1], iMin, iMax, eNull, &rval, 0, &op);
if( op==VEC1_OP_EQ ){
aRval[nRval++] = rval;
}
}
op = VEC1_OP_IN;
rval = nRval;
*paRval = aRval;
}
}else if( pFilter->vop==VEC1_OP_NOTNULL ){
res = 1;
}else if( pFilter->eType==SQLITE_NULL && pFilter->vop==VEC1_OP_IS ){
rval = (int)eNull;
op = VEC1_OP_EQ;
}else if( pFilter->eType!=SQLITE_NULL ){
i64 iVal = pFilter->iVal;
if( pFilter->eType==SQLITE_TEXT || pFilter->eType==SQLITE_BLOB ){
/* All text and blob values are greater than all 1-byte ints */
iVal = iMax+10;
}else if( pFilter->eType==SQLITE_FLOAT ){
iVal = (i64)pFilter->fVal;
if( (double)iVal!=pFilter->fVal ){
if( pFilter->vop==VEC1_OP_EQ || pFilter->vop==VEC1_OP_IS ){
iVal = iMin-10;
}else if( pFilter->vop==VEC1_OP_LT || pFilter->vop==VEC1_OP_GE ){
iVal = iVal+1;
}
}
}
if( iVal<iMin ){
res = (pFilter->vop==VEC1_OP_GT || pFilter->vop==VEC1_OP_GE);
}else if( iVal>iMax ){
res = (pFilter->vop==VEC1_OP_LT || pFilter->vop==VEC1_OP_LE);
}else{
rval = (int)iVal;
op = (pFilter->vop==VEC1_OP_IS ? VEC1_OP_EQ : pFilter->vop);
}
}
if( res>0 ){
assert( eNull==(iMax+1) );
op = VEC1_OP_LT;
rval = (int)eNull;
}
*pRval = rval;
*pOp = op;
return SQLITE_OK;
}
static void vec1MetaFilterIntTail(
const u8 *aMeta,
int iBitOff,
u8 *aBitmask,
int nEntry,
int szInt,
int rval,
int *aRval,
u8 op
){
const int eNull = szInt==1 ? VEC1_META_1BYTENULL : VEC1_META_4BYTENULL;
int jj;
assert( szInt==1 || szInt==4 );
for(jj=0; jj<nEntry; jj++){
int bSet = 0;
int iVal;
if( szInt==1 ){
iVal = (int)aMeta[jj];
}else{
iVal = (int)vec1GetU32(&aMeta[jj*sizeof_u32]);
}
switch( op ){
case VEC1_OP_EQ:
bSet = (iVal==rval);
break;
case VEC1_OP_GT:
bSet = (iVal>rval) && iVal!=eNull;
break;
case VEC1_OP_LT:
bSet = (iVal<rval);
break;
case VEC1_OP_GE:
bSet = (iVal>=rval) && iVal!=eNull;
break;
case VEC1_OP_LE:
bSet = (iVal<=rval);
break;
default: {
int ii;
assert( op==VEC1_OP_IN );
for(ii=0; ii<rval; ii++){
if( iVal==aRval[ii] ){
bSet = 1;
break;
}
}
break;
}
}
if( bSet==0 ){
int ii = jj+iBitOff;
aBitmask[ii / 8] |= (1 << (ii % 8));
}
}
}
static void vec1MetaFilterScan1ByteArray(
const u8 *a,
u8 *aBitmask,
int nEntry,
int rval, int *aRval, u8 op
){
int jj = 0;
#if defined(VEC1_HAVE_AVX2)
__m256i rhs = _mm256_set1_epi8((char)rval);
__m256i sign = _mm256_set1_epi8((char)0x80);
__m256i null = _mm256_set1_epi8((char)VEC1_META_1BYTENULL);
for(; jj<(nEntry-31); jj+=32){
__m256i lhs = _mm256_loadu_si256((__m256i*)&a[jj]);
u32 mask = 0;
u32 prev = 0;
switch( op ){
case VEC1_OP_EQ: {
__m256i cmp = _mm256_cmpeq_epi8(lhs, rhs);
mask = (u32)_mm256_movemask_epi8(cmp);
break;
}
case VEC1_OP_GT: {
__m256i cmp = _mm256_cmpgt_epi8(
_mm256_xor_si256(sign, lhs),
_mm256_xor_si256(sign, rhs)
);
mask = (u32)_mm256_movemask_epi8(cmp);
cmp = _mm256_cmpeq_epi8(lhs, null);
mask &= ~(u32)_mm256_movemask_epi8(cmp);
break;
}
case VEC1_OP_LT: {
__m256i cmp = _mm256_cmpgt_epi8(
_mm256_xor_si256(sign, rhs),
_mm256_xor_si256(sign, lhs)
);
mask = (u32)_mm256_movemask_epi8(cmp);
break;
}
case VEC1_OP_GE: {
__m256i cmp = _mm256_cmpgt_epi8(
_mm256_xor_si256(sign, rhs),
_mm256_xor_si256(sign, lhs)
);
mask = ~(u32)_mm256_movemask_epi8(cmp);
cmp = _mm256_cmpeq_epi8(lhs, null);
mask &= ~(u32)_mm256_movemask_epi8(cmp);
break;
}
case VEC1_OP_LE: {
__m256i cmp = _mm256_cmpgt_epi8(
_mm256_xor_si256(sign, lhs),
_mm256_xor_si256(sign, rhs)
);
mask = ~(u32)_mm256_movemask_epi8(cmp);
break;
}
default: {
int ii;
assert( op==VEC1_OP_IN );
for(ii=0; ii<rval; ii++){
__m256i rhs2 = _mm256_set1_epi8((char)aRval[ii]);
__m256i cmp = _mm256_cmpeq_epi8(lhs, rhs2);
mask |= (u32)_mm256_movemask_epi8(cmp);
}
break;
}
}
memcpy(&prev, &aBitmask[jj/8], sizeof_u32);
mask = (~mask | prev);
memcpy(&aBitmask[jj/8], &mask, sizeof_u32);
}
#endif
#if defined(VEC1_HAVE_NEON)
uint8x16_t rhs_n = vdupq_n_u8((uint8_t)rval);
uint8x16_t sign_n = vdupq_n_u8(0x80);
uint8x16_t null_n = vdupq_n_u8((uint8_t)VEC1_META_1BYTENULL);
for(; jj<(nEntry-15); jj+=16){
uint8x16_t lhs = vld1q_u8((const uint8_t*)&a[jj]);
uint8x16_t cmp;
u8 m1 = 0;
u8 m2 = 0;
switch( op ){
case VEC1_OP_IN: {
int ii;
cmp = vdupq_n_u8(0);
for(ii=0; ii<rval; ii++){
uint8x16_t r2 = vdupq_n_u8((uint8_t)aRval[ii]);
cmp = vorrq_u8(cmp, vceqq_u8(lhs, r2));
}
break;
}
case VEC1_OP_EQ: {
cmp = vceqq_u8(lhs, rhs_n);
break;
}
case VEC1_OP_GT: {
cmp = vcgtq_u8(lhs, rhs_n);
cmp = vbicq_u8(cmp, vceqq_u8(lhs, null_n));
break;
}
case VEC1_OP_LT: {
cmp = vcltq_u8(lhs, rhs_n);
break;
}
case VEC1_OP_GE: {
cmp = vcgeq_u8(lhs, rhs_n);
cmp = vbicq_u8(cmp, vceqq_u8(lhs, null_n));
break;
}
case VEC1_OP_LE: {
cmp = vcleq_u8(lhs, rhs_n);
break;
}
default:
cmp = vdupq_n_u8(0);
break;
}
{
static const uint8_t weights[8] = {1,2,4,8,16,32,64,128};
uint8x8_t w = vld1_u8(weights);
uint8x8_t lo = vget_low_u8(cmp);
uint8x8_t hi = vget_high_u8(cmp);
lo = vshr_n_u8(lo, 7); /* 0xFF->0x01 */
hi = vshr_n_u8(hi, 7);
m1 = (u16)vaddlvq_u16(vmull_u8(lo, w));
m2 = (u16)vaddlvq_u16(vmull_u8(hi, w));
}
aBitmask[(jj/8)] |= ~m1;
aBitmask[(jj/8)+1] |= ~m2;
}
#endif
vec1MetaFilterIntTail(
&a[jj], jj, aBitmask, nEntry-jj, 1, rval, aRval, op
);
}
static void vec1MetaFilterScan4ByteArray(
const int *a,
u8 *aBitmask,
int nEntry,
int rval, int *aRval, u8 op
){
int jj = 0;
#if defined(VEC1_HAVE_AVX2)
const __m256i rhs = _mm256_set1_epi32(rval);
const __m256i null = _mm256_set1_epi32(VEC1_META_4BYTENULL);
/* Shuffle mask used to swap the byte-order of the 32-bit integers
** in this meta-data list if required. There are no big-endian AVX2
** systems, so no need for a runtime check here. */
const __m256i swap = _mm256_setr_epi8(
3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12,
3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12
);
for(; jj<(nEntry-7); jj+=8){
__m256i lhs = _mm256_loadu_si256((__m256i*)&a[jj]);
u8 mask = 0;
/* Shuffle the 8 32-bit int values to little-endian order */
lhs = _mm256_shuffle_epi8(lhs, swap);
switch( op ){
case VEC1_OP_EQ: {
__m256i cmp = _mm256_cmpeq_epi32(lhs, rhs);
mask = (u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
break;
}
case VEC1_OP_GT: {
__m256i cmp = _mm256_cmpgt_epi32(lhs, rhs);
mask = (u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
cmp = _mm256_cmpeq_epi32(lhs, null);
mask &= ~(u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
break;
}
case VEC1_OP_LT: {
__m256i cmp = _mm256_cmpgt_epi32(rhs, lhs);
mask = (u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
break;
}
case VEC1_OP_GE: {
__m256i cmp = _mm256_cmpgt_epi32(rhs, lhs);
mask = ~(u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
cmp = _mm256_cmpeq_epi32(lhs, null);
mask &= ~(u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
break;
}
case VEC1_OP_LE: {
__m256i cmp = _mm256_cmpgt_epi32(lhs, rhs);
mask = ~(u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
break;
}
default: {
int ii;
assert( op==VEC1_OP_IN );
for(ii=0; ii<rval; ii++){
__m256i rhs2 = _mm256_set1_epi32(aRval[ii]);
__m256i cmp = _mm256_cmpeq_epi32(lhs, rhs2);
mask |= (u8)_mm256_movemask_ps(_mm256_castsi256_ps(cmp));
}
break;
}
}
mask = ~mask;
aBitmask[jj/8] |= mask;
}
#endif
#if defined(VEC1_HAVE_NEON)
const int32x4_t rhs = vdupq_n_s32(rval);
const int32x4_t null = vdupq_n_s32(VEC1_META_4BYTENULL);
for(; jj<(nEntry-3); jj+=4){
uint32x4_t cmp = vdupq_n_u32(0);
int32x4_t lhs =
vreinterpretq_s32_u8(vrev32q_u8(vld1q_u8((uint8_t*)&a[jj]))
);
/* cmp is a vector of 4 u32 values. This block sets each value to
** 0xFFFFFFFF if the constraint is true for the corresponding meta-data
** value (i.e. if the row will not be filtered out) or to 0x00000000
** otherwise. */
switch( op ){
case VEC1_OP_IN: {
int ii;
for(ii=0; ii<rval; ii++){
int32x4_t rhs2 = vdupq_n_s32(aRval[ii]);
cmp = vorrq_u32(cmp, vceqq_s32(lhs, rhs2));
}
break;
}
case VEC1_OP_EQ: {
cmp = vceqq_s32(lhs, rhs);
break;
}
case VEC1_OP_GT: {
cmp = vcgtq_s32(lhs, rhs);
cmp = vbicq_u32(cmp, vceqq_s32(lhs, null));
break;
}
case VEC1_OP_LT: {
cmp = vcltq_s32(lhs, rhs);
break;
}
case VEC1_OP_GE: {
cmp = vcgeq_s32(lhs, rhs);
cmp = vbicq_u32(cmp, vceqq_s32(lhs, null));
break;
}
case VEC1_OP_LE: {
cmp = vcleq_s32(lhs, rhs);
break;
}
}
/* Collapse 4 x 32-bit mask lanes to a 4-bit nibble.
** NEON has no movemask, so shift the high bit of each lane
** into position and OR them together. */
u32 mask = (u8)(
(vgetq_lane_u32(cmp, 0) & 0x01) |
(vgetq_lane_u32(cmp, 1) & 0x02) |
(vgetq_lane_u32(cmp, 2) & 0x04) |
(vgetq_lane_u32(cmp, 3) & 0x08)
);
mask = (~mask) & 0xF;
aBitmask[jj/8] |= (jj & 4) ? (mask << 4) : mask;
}
#endif
vec1MetaFilterIntTail(
(const u8*)&a[jj], jj, aBitmask, nEntry-jj, 4, rval, aRval, op
);
}
/*
** Buffer pMeta contains an nEntry entry meta-value list in either
** 1BYTEINT (if szInt==1) or 4BYTEINT (if szInt==4) format. This function
** runs filter pFilter over the list, and sets the corresponding bit in
** pBitmask for each entry that does not match the filter (i.e. that
** should be excluded from the query results).
**
** SQLITE_OK is returned if successful, or an error code otherwise.
*/
static int vec1MetaFilterIntList(
Vec1Filter *pFilter,
int szInt,
int nEntry,
Vec1Buffer *pMeta,
Vec1Buffer *pBitmask
){
int rc = SQLITE_OK;
if( pMeta->n<(VEC1_META_SZHDR+(nEntry*szInt)) ){
rc = VEC1_CORRUPT;
}else{
int *aRval = 0;
int rval = 0;
u8 op = 0;
rc = vec1FilterToIntOp(pFilter,
(szInt==4 ? VEC1_META_4BYTEMIN : 0),
(szInt==4 ? VEC1_META_4BYTEMAX : 254),
(szInt==4 ? VEC1_META_4BYTENULL : VEC1_META_1BYTENULL),
&rval, &aRval, &op
);
if( rc==SQLITE_OK ){
if( op==0 ){
/* No entries will match. Set the "filtered-out" bit for all rows. */
memset(pBitmask->a, 0xFF, pBitmask->n);
}else if( szInt==4 ){
const int *a = (const int*)&pMeta->a[VEC1_META_SZHDR];
vec1MetaFilterScan4ByteArray(a, pBitmask->a, nEntry, rval, aRval, op);
}else{
assert( szInt==1 );
const u8 *a = (const u8*)&pMeta->a[VEC1_META_SZHDR];
vec1MetaFilterScan1ByteArray(a, pBitmask->a, nEntry, rval, aRval, op);
}
}
sqlite3_free(aRval);
}
return rc;
}
static VEC1_NOINLINE int vec1DoMetaFilters(
Vec1Tab *pTab,
Vec1Query *pQuery,
i64 iId,
int nEntry,
Vec1Buffer *pMeta,
Vec1Buffer *pBitmask
){
int rc = SQLITE_OK;
int ii;
/* Grow and zero the bitmask array. */
int nByte = (nEntry+7)/8;
pBitmask->n = 0;
rc = vec1BufferGrow(pBitmask, nByte);
if( rc==SQLITE_OK ){
memset(pBitmask->a, 0, nByte);
}
pBitmask->n = nByte;
assert( pQuery->nFilter>0 );
for(ii=0; rc==SQLITE_OK && ii<pQuery->nFilter; ii++){
Vec1Filter *pFilter = &pQuery->aFilter[ii];
VEC1_QINSTR_START(pTab, VEC1_QINSTR_METAREAD);
rc = vec1ReadMeta(pTab, nEntry, pMeta, iId, pFilter->iMeta);
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_METAREAD);
VEC1_QINSTR_START(pTab, VEC1_QINSTR_METASCAN);
if( rc==SQLITE_OK ){
int jj = 0;
u32 f = vec1GetU32(pMeta->a);
if( f & VEC1_META_1BYTEINT ){
/* Single-byte integer format */
rc = vec1MetaFilterIntList(pFilter, 1, nEntry, pMeta, pBitmask);
}else if( f & VEC1_META_4BYTEINT ){
/* 4-byte integer format */
rc = vec1MetaFilterIntList(pFilter, 4, nEntry, pMeta, pBitmask);
}else if( f & VEC1_META_REAL ){
Vec1MetaValue val = {0, 0, 0, 0};
for(jj=0; jj<nEntry; jj++){
u64 iVal = vec1GetU64(
&pMeta->a[VEC1_META_SZHDR+jj*sizeof_f64]
);
if( iVal==VEC1_META_REALNULL ){
val.eType = SQLITE_NULL;
}else{
val.eType = SQLITE_FLOAT;
memcpy(&val.fVal, &iVal, sizeof(u64));
}
if( vec1MetaValueFilter(pFilter, &val) ){
pBitmask->a[jj / 8] |= (1 << (jj % 8));
}
}
}else{
/* Generic format */
int iOff = VEC1_META_SZHDR;
for(jj=0; jj<nEntry; jj++){
Vec1MetaValue val;
rc = vec1MetaValueRead(pMeta, &iOff, &val);
if( rc!=SQLITE_OK ) break;
if( vec1MetaValueFilter(pFilter, &val) ){
pBitmask->a[jj / 8] |= (1 << (jj % 8));
}
}
}
}
if( pFilter->vop==VEC1_OP_IN ){
ii += (int)pFilter->iVal;
}
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_METASCAN);
}
return rc;
}
static void vec1EncodeVectorBit(
const Vec1Model *pMod, /* Model */
const float *aVec, /* Input vector */
u8 *aCode /* OUT: Encoded vector */
){
const int nElem = (int)pMod->hdr.nElem;
float norm;
int ii;
u8 *pOut;
assert( pMod->hdr.flags & VEC1_MODEL_BITQUANT );
norm = (float)vec1VectorNorm2(aVec, nElem);
norm = sqrtf(norm);
memcpy(aCode, &norm, sizeof_f32);
assert( pMod->nCodeElem==8 || pMod->nCodeElem==16 );
pOut = &aCode[4];
for(ii=0; ii<nElem; ii+=pMod->nCodeElem){
u8 v = 0;
int jj;
if( pMod->nCodeElem==8 ){
for(jj=0; jj<8; jj++){
if( aVec[ii+jj]>0.0 ){
v |= (1<<jj);
}
}
}else{
for(jj=0; jj<8; jj+=1){
if( (aVec[ii+jj*2]+aVec[ii+jj*2+1])>0.0 ){
v |= (1<<jj);
}
}
}
*pOut = v;
pOut++;
}
}
static void vec1EncodeVectorPQ(
const Vec1Model *pMod,
const float *aVec,
u8 *aCode
){
const int nCodebook = pMod->hdr.nCodebook;
const int nCodeElem = pMod->nCodeElem;
const float *pIn = pMod->aModelT;
int M, K;
for(M=0; M<nCodebook; M++){
float fBestDist = INFINITY;
for(K=0; K<VEC1_PQ_CODEBOOK_SZ; K+=VEC1_TRANSPOSE_WIDTH){
int ii;
float aAcc[VEC1_TRANSPOSE_WIDTH];
vec1ModelTDist(pIn, &aVec[M*nCodeElem], nCodeElem, aAcc);
pIn += (VEC1_TRANSPOSE_WIDTH * nCodeElem);
for(ii=0; ii<VEC1_TRANSPOSE_WIDTH; ii++){
if( aAcc[ii]<fBestDist ){
fBestDist = aAcc[ii];
aCode[M] = (u8)(K+ii);
}
}
}
}
}
static void vec1EncodeVectorPQSimple(
const Vec1Model *pMod, /* Model to encode with */
const float *aVec, /* Vector to encode */
u8 *aOut, /* OUT: Quantized vector */
double *pfTotalDist /* OUT: reconstruction L2 squared from orig */
){
const int nCodebook = (int)pMod->hdr.nCodebook;
const int nCodeElem = pMod->nCodeElem;
int ii = 0;
for(ii=0; ii<nCodebook; ii++){
const float *pCodebook = &pMod->aModel[ii*nCodeElem*VEC1_PQ_CODEBOOK_SZ];
const float *aSub = &aVec[ii*nCodeElem];
aOut[ii] = (u8)vec1BestMatchSimple(
pCodebook, VEC1_PQ_CODEBOOK_SZ, aSub, nCodeElem, pfTotalDist
);
}
}
static void vec1EncodeVector(
const Vec1Model *pMod,
const float *aVec,
u8 *aCode
){
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ){
vec1EncodeVectorBit(pMod, aVec, aCode);
}else{
vec1EncodeVectorPQ(pMod, aVec, aCode);
}
}
/*
** Buffer aCode[] contains a vector quantized according to model pMod.
** This function (lossily) reconstructs the original vector and writes
** it to buffer aVec[] before returning.
*/
static void vec1DecodeVector(
const Vec1Model *pMod, /* Model used to encode vector */
const u8 *aCode, /* Encoded vector */
float *aVec /* Populate this buffer with full vector */
){
const int nElem = (int)pMod->hdr.nElem;
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ){
float fNorm = 0.0;
float fVal = 0.0;
int ii;
/* The first 4 bytes of the encoded vector store the norm of the vector,
** as a 32-bit float. */
memcpy(&fNorm, aCode, sizeof_f32);
/* The reconstructed vector uses the following value as the magnitude
** of each dimension. The value is -ve if the corresponding bit is clear,
** or +ve if it is set. */
fVal = (float)sqrt((fNorm*fNorm) / nElem);
assert( pMod->nCodeElem==8 || pMod->nCodeElem==16 );
for(ii=0; ii<nElem; ii++){
if( pMod->nCodeElem==8 ){
if( aCode[sizeof_f32 + (ii/8)] & (1 << (ii%8)) ){
aVec[ii] = fVal;
}else{
aVec[ii] = -1 * fVal;
}
}else{
if( aCode[sizeof_f32 + (ii/16)] & (1 << ((ii/2)%8)) ){
aVec[ii] = fVal;
}else{
aVec[ii] = -1 * fVal;
}
}
}
}else{
int ii;
for(ii=0; ii<(int)pMod->hdr.nCodebook; ii++){
const float *aCopy = &pMod->aModel[
(ii * VEC1_PQ_CODEBOOK_SZ + aCode[ii]) * pMod->nCodeElem
];
int nCopy = MIN(pMod->nCodeElem, nElem - ii*pMod->nCodeElem);
memcpy(&aVec[ii * pMod->nCodeElem], aCopy, nCopy*sizeof_f32);
}
}
}
/*
** Return true if any value in aAcc[] is less than fMin. False otherwise.
*/
static int vec1AnyLessThan(const float aAcc[16], float fMin){
#if defined(VEC1_HAVE_AVX2)
__m256 vMin = _mm256_set1_ps(fMin);
__m256 cmp0 = _mm256_cmp_ps(_mm256_loadu_ps(aAcc), vMin, _CMP_LT_OQ);
__m256 cmp1 = _mm256_cmp_ps(_mm256_loadu_ps(aAcc+8), vMin, _CMP_LT_OQ);
__m256 mask = _mm256_or_ps(cmp0, cmp1);
return _mm256_movemask_ps(mask)!=0;
#elif defined(VEC1_HAVE_NEON)
float32x4_t vMin = vdupq_n_f32(fMin);
uint32x4_t cmp0 = vcltq_f32(vld1q_f32(aAcc), vMin);
uint32x4_t cmp1 = vcltq_f32(vld1q_f32(aAcc+4), vMin);
uint32x4_t cmp2 = vcltq_f32(vld1q_f32(aAcc+8), vMin);
uint32x4_t cmp3 = vcltq_f32(vld1q_f32(aAcc+12), vMin);
uint32x4_t mask = vorrq_u32(vorrq_u32(cmp0, cmp1), vorrq_u32(cmp2, cmp3));
return vmaxvq_u32(mask)!=0;
#else
int ii;
for(ii=0; ii<16; ii++){
if( aAcc[ii]<fMin ) return 1;
}
return 0;
#endif
}
static void vec1ScanBlocked(
Vec1AnnHeap *pHeap,
u8 *aMask,
const float *aDist,
int nCodebook,
const u8 *aBlob, int nBlob,
int szRowid,
int nEntry,
int nNorm,
float fResidualNorm2
){
const int szBlk = VEC1_PQ_BLOCKSIZE * (nCodebook + nNorm);
int iOff = 0;
int iBlk = 0;
int iEarly = nCodebook;
if( nNorm==0 && pHeap->bStreaming==0 ){
iEarly = (nCodebook/4);
}
assert( nNorm==sizeof_f32 || nNorm==0 );
/* One iteration of this loop for each block in the blob. */
for(iOff=VEC1_LIST_SZHDR + nEntry*szRowid;
iOff<nBlob;
iOff+=szBlk, iBlk++
){
int ii;
const u8 *aBlk = &aBlob[iOff];
const u8 *aRowid = &aBlob[VEC1_LIST_SZHDR + iBlk*VEC1_PQ_BLOCKSIZE*szRowid];
int nSlot = MIN(VEC1_PQ_BLOCKSIZE, (nEntry - iBlk*VEC1_PQ_BLOCKSIZE));
int iBook;
float aAcc[VEC1_PQ_BLOCKSIZE];
memset(aAcc, 0, sizeof(aAcc));
for(iBook=0; iBook<nCodebook; iBook++){
const float *aTable = &aDist[iBook * VEC1_PQ_CODEBOOK_SZ];
const u8 *aCode = &aBlk[(iBook + nNorm) * VEC1_PQ_BLOCKSIZE];
int jj;
for(jj=0; jj<VEC1_PQ_BLOCKSIZE; jj++){
aAcc[jj] += aTable[ aCode[jj] ];
}
/* If all 16 distances sums are already greater than the required
** minimum, quit processing this block early. */
if( iBook>=iEarly && (iBook & 0x03)==0x03
&& vec1AnyLessThan(aAcc, (float)pHeap->fMin)==0
){
goto skip;
}
}
if( nNorm ){
int jj;
for(jj=0; jj<nSlot; jj++){
float norm;
memcpy(&norm, &aBlk[jj*sizeof_f32], sizeof_f32);
aAcc[jj] = (float)(fResidualNorm2 + (norm - 2.0 * aAcc[jj]) * norm);
}
}
for(ii=0; ii<nSlot; ii++){
float dist = aAcc[ii];
if( dist<pHeap->fMin
&& (aMask==0 || (aMask[(iBlk*VEC1_PQ_BLOCKSIZE+ii)/8] & (1 << (ii%8)))==0)
){
i64 iRowid = 0;
if( szRowid==4 ){
iRowid = vec1GetU32(&aRowid[ii * szRowid]);
if( iRowid==VEC1_TOMBSTONE_32 ) continue;
}else{
iRowid = vec1GetU64(&aRowid[ii * szRowid]);
if( iRowid==VEC1_TOMBSTONE_64 ) continue;
}
vec1HeapInsert(pHeap, iRowid, dist);
}
}
skip:; /* semi-colon required to avoid C23 warning/error */
}
}
static int vec1DoKANNBucket(
Vec1Query *pQuery,
int iBucket,
Vec1Buffer *pDist,
Vec1Buffer *pMeta,
Vec1Buffer *pBitmask
){
Vec1Tab *pTab = pQuery->pTab;
Vec1AnnHeap *pHeap = &pQuery->heap;
const int nElem = pTab->cfg.nElem;
const int bUseBQ = (pTab->mod.hdr.flags & VEC1_MODEL_BITQUANT) ? 1 : 0;
const int nCodebook = (int)pTab->mod.hdr.nCodebook;
const int bUsePQ = (nCodebook>0 && bUseBQ==0);
sqlite3_stmt *pScanIdx = 0;
int rc = SQLITE_OK;
float *aDist = 0;
float fResidualNorm2 = 0.0;
if( iBucket<0 ){
rc = vec1GetSql(pTab, VEC1_SQL_SCAN_IDX, &pScanIdx);
}else{
rc = vec1GetSql(pTab, VEC1_SCAN_BUCKET, &pScanIdx);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pScanIdx, 1, iBucket);
}
}
assert( 0==pDist->n || 0==(pTab->mod.hdr.flags & VEC1_MODEL_RESIDUAL) );
if( (bUsePQ || bUseBQ) ){
/* If using PQ or BQ codes, build a LUT for this bucket. */
if( rc==SQLITE_OK && pDist->n==0 ){
const float *aEncode = pQuery->aTransform;
int nByte = (bUsePQ ?
vec1PQLUTSize(nCodebook) : /* Product quant */
vec1BinaryQuantSize(nElem) /* Binary quant */
);
rc = vec1BufferSize(pDist, nByte);
if( rc!=SQLITE_OK ) return rc;
aDist = (float*)pDist->a;
VEC1_QINSTR_START(pTab, VEC1_QINSTR_LUT);
if( pTab->mod.hdr.flags & VEC1_MODEL_RESIDUAL ){
float *aTmp = pTab->aTmpVec;
assert( iBucket>=0 && iBucket<(int)pTab->mod.hdr.nBucket );
vec1Sub(aTmp, aEncode, &pTab->mod.aCentroid[iBucket*nElem], nElem);
aEncode = aTmp;
}
if( bUsePQ ){
vec1PQLUTBuild(pTab, aEncode, aDist);
}else{
vec1BinaryQuantLUTBuild(pTab, aEncode, aDist);
fResidualNorm2 = (float)vec1VectorNorm2(aEncode, pTab->cfg.nElem);
vec1EncodeVectorBit(&pTab->mod, aEncode, pQuery->aBit);
}
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_LUT);
pDist->n = nByte;
}
aDist = (float*)pDist->a;
}
while( rc==SQLITE_OK ){
i64 iId = 0;
int res;
/* Load the next index entry from the %_idx table */
VEC1_QINSTR_START(pTab, VEC1_QINSTR_IDXREAD);
res = sqlite3_step(pScanIdx);
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_IDXREAD);
if( res!=SQLITE_ROW ) break;
const u8 *aBlob = (const u8*)sqlite3_column_blob(pScanIdx, 0);
int nBlob = sqlite3_column_bytes(pScanIdx, 0);
int nEntry = 0;
int flags = 0;
int ii;
int szRowid = 4;
if( vec1CheckIdxSize(pTab, aBlob, nBlob) ){
rc = VEC1_CORRUPT;
break;
}
nEntry = vec1GetU32(&aBlob[4]);
iId = sqlite3_column_int64(pScanIdx, 2);
if( pQuery->nFilter>0 ){
rc = vec1DoMetaFilters(pTab, pQuery, iId, nEntry, pMeta, pBitmask);
if( rc!=SQLITE_OK ) break;
}
/* Loop through each of the PQ vectors in aBlob[] */
VEC1_QINSTR_START(pTab, VEC1_QINSTR_PQSCAN);
flags = vec1GetU32(&aBlob[0]);
if( flags & VEC1_LIST_64BIT ) szRowid = 8;
if( aDist ){
vec1ScanBlocked(
pHeap, pBitmask->a, aDist, nCodebook, aBlob, nBlob, szRowid,
nEntry, (bUseBQ ? sizeof_f32 : 0), fResidualNorm2
);
}else{
const int szVec = sizeof_f32*nElem;
assert( bUseBQ==0 && bUsePQ==0 );
for(ii=0; ii<nEntry; ii++){
int iRowidOff = VEC1_LIST_SZHDR + ii*szRowid;
int iVecOff = VEC1_LIST_SZHDR + nEntry*szRowid + ii*szVec;
const float *aFull = (const float*)&aBlob[iVecOff];
double fDist = 0.0;
i64 iRowid;
/* Check if meta-value filters excluded this row. If so, skip
** directly to the next iteration of the loop. */
if( pBitmask->n>0 && (pBitmask->a[ii/8] & (1 << (ii % 8))) ){
continue;
}
if( szRowid==4 ){
iRowid = vec1GetU32(&aBlob[iRowidOff]);
if( iRowid==VEC1_TOMBSTONE_32 ) continue;
}else{
iRowid = vec1GetU64(&aBlob[iRowidOff]);
if( iRowid==VEC1_TOMBSTONE_64 ) continue;
}
if( pTab->mod.hdr.eDistance==VEC1_DISTANCE_COS ){
fDist = vec1CosDist(pQuery->aVector, aFull, nElem);
}else{
fDist = vec1L2Dist(pQuery->aVector, aFull, nElem);
}
vec1HeapInsert(pHeap, iRowid, fDist);
}
}
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_PQSCAN);
}
vec1StmtReset(&rc, pScanIdx);
if( rc==SQLITE_OK ) rc = pHeap->rc;
return rc;
}
static int vec1DoKANNQuery(Vec1Csr *pCsr){
int rc = SQLITE_OK;
Vec1Tab *pTab = (Vec1Tab*)(pCsr->base.pVtab);
Vec1Query *pQuery = pCsr->pQuery;
Vec1AnnHeap *pHeap = &pQuery->heap;
/* Table of distances used for ANN queries against the PQ quantized
** vectors in the %_idx table. This is an array of (nCodebook * 256)
** float values. The 256 distances for each codebook are stored
** contiguously, so to lookup code iCode in codebook iBook:
**
** aDist[(iBook * 256) + iCode]
*/
Vec1BucketResult *aBucket = 0;
const float *aVector = pQuery->aVector;
int iProbe = 0;
Vec1Buffer meta = {0, 0, 0}; /* Buffer to load meta arrays */
Vec1Buffer bitmask = {0, 0, 0}; /* Buffer for meta filter bitmask */
Vec1Buffer dist = {0, 0, 0}; /* Buffer for LUT */
VEC1_QINSTR_START(pTab, VEC1_QINSTR_INITQUERY);
/* Transform the input vector if required. If no transformation is
** required, then pQuery->aTransform already points to the same buffer
** as pQuery->aVector. */
assert( vec1TransformRequired(&pTab->mod)==(pQuery->aTransform!=aVector) );
vec1TransformInputVector(&pTab->mod, pQuery->aTransform, aVector);
rc = vec1HeapInit(pHeap, pQuery->K, pQuery->bStreaming);
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_INITQUERY);
/* Do the coarse quantizer step. If nbucket is not 0, this block allocates
** the aBucket[] array. The first nProbe elements of which are the ones
** that should be scanned by this call. The others may be used later if
** this is a streaming query, but are usually ignored. */
VEC1_QINSTR_START(pTab, VEC1_QINSTR_COARSE);
if( rc==SQLITE_OK ){
if( pTab->mod.hdr.nBucket==0 ){
assert( pQuery->nProbe==1 );
}else{
assert( pQuery->nProbe>=1 && pQuery->nProbe<=(int)pTab->mod.hdr.nBucket );
rc = vec1FindBuckets(pTab, pQuery->aTransform, pQuery->nProbe, &aBucket);
}
}
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_COARSE);
for(iProbe=0; iProbe<pQuery->nProbe && rc==SQLITE_OK; iProbe++){
int iBucket = -1;
if( pTab->mod.hdr.nBucket>0 ){
iBucket = aBucket[iProbe].iBucket;
if( pQuery->nProbeSlack>0.0
&& aBucket[iProbe].fDist>aBucket[0].fDist*(1.0+pQuery->nProbeSlack)
){
break;
}
}
if( pTab->mod.hdr.flags & VEC1_MODEL_RESIDUAL ){
dist.n = 0;
}
rc = vec1DoKANNBucket(pQuery, iBucket, &dist, &meta, &bitmask);
}
VEC1_QINSTR_START(pTab, VEC1_QINSTR_FINALSORT);
if( rc==SQLITE_OK ){
i64 nSort = MIN( pHeap->nRes, pQuery->K );
if( pQuery->bStreaming ){
/* For a streaming query, set the pQuery->aBucket[] array to contain
** the remaining buckets - the ones that have not yet been scanned. */
pQuery->nBucket = pTab->mod.hdr.nBucket - iProbe;
if( pQuery->nBucket>0 ){
int nCopy = sizeof(Vec1BucketResult) * pQuery->nBucket;
memmove(aBucket, &aBucket[iProbe], nCopy);
pQuery->aBucket = aBucket;
aBucket = 0;
}
pQuery->nOrigRes = (int)MAX(16, pHeap->nRes);
}
/* For a streaming query, all candidate vectors were added to the
** aRes[] array. The heap has guaranteed that the first nSort elements
** of the Vec1AnnHeap.aRes[] array have distances <= than the remaining
** elements. So it is still correct just to sort the first nSort
** elements of aRes[] and set nCurrentRes=nSort to configure the first
** block of rows to return. */
rc = vec1AnnResultSort(pHeap->aRes, nSort);
pCsr->nCurrentRes = (int)nSort;
}
sqlite3_free(aBucket);
vec1BufferFree(&meta);
vec1BufferFree(&bitmask);
vec1BufferFree(&dist);
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_FINALSORT);
return rc;
}
/*
** Load the contents of the %_config table into the pTab->cfg structure.
** if the iModelVersion field has changed since %_config was last loaded,
** also load the model from the %_model table.
*/
static int vec1LoadConfig(Vec1Tab *pTab){
sqlite3_stmt *pStmt = 0;
int rc = SQLITE_OK;
int nReq = 0;
pTab->cfg.nBlocksize = VEC1_BLOCKSIZE_DEFAULT;
pTab->cfg.nBlocksizeMin = VEC1_BLOCKSIZE_MIN_DEFAULT;
pTab->cfg.nElem = 0;
rc = vec1GetSql(pTab, VEC1_SQL_LOAD_CONFIG, &pStmt);
if( rc==SQLITE_OK ){
while( sqlite3_step(pStmt)==SQLITE_ROW ){
int iField = sqlite3_column_int(pStmt, 0);
switch( iField ){
case VEC1_CONFIG_MODEL: {
i64 iNew = sqlite3_column_int64(pStmt, 1);
if( pTab->cfg.iModelVersion!=iNew ){
rc = vec1LoadModel(pTab);
if( rc==SQLITE_OK ){
pTab->cfg.iModelVersion = iNew;
}
}
break;
}
case VEC1_CONFIG_BLOCKSIZE: {
pTab->cfg.nBlocksize = sqlite3_column_int(pStmt, 1);
pTab->cfg.nBlocksize = MAX(VEC1_BLOCKSIZE_MIN, pTab->cfg.nBlocksize);
pTab->cfg.nBlocksize = MIN(VEC1_BLOCKSIZE_MAX, pTab->cfg.nBlocksize);
break;
}
case VEC1_CONFIG_BLOCKSIZE_MIN: {
int nMin = sqlite3_column_int(pStmt, 1);
nMin = MAX(VEC1_BLOCKSIZE_MIN, nMin);
nMin = MIN(VEC1_BLOCKSIZE_MAX, nMin);
pTab->cfg.nBlocksizeMin = nMin;
break;
}
case VEC1_CONFIG_NELEM: {
pTab->cfg.nElem = sqlite3_column_int(pStmt, 1);
break;
}
case VEC1_CONFIG_FMTVERSION: {
int iVersion = sqlite3_column_int(pStmt, 1);
if( iVersion<VEC1_CURRENT_FMTVERSION ){
vec1VtabError(pTab, "unrecognized index version");
rc = SQLITE_ERROR;
}else if( iVersion>VEC1_CURRENT_FMTVERSION ){
vec1VtabError(pTab,
"unsupported index version - requires vec1 %d.%d",
(iVersion / 1000), (iVersion % 1000)
);
rc = SQLITE_ERROR;
}
break;
}
default: {
vec1VtabError(pTab, "unrecognized config entry: %d", iField);
rc = SQLITE_ERROR;
break;
}
}
}
vec1StmtReset(&rc, pStmt);
}
nReq = MAX(pTab->mod.hdr.nElem, pTab->mod.nCodeElem*pTab->mod.hdr.nCodebook);
if( (rc==SQLITE_OK) && nReq>0 && nReq!=pTab->nTmpVec ){
sqlite3_free(pTab->aTmpVec);
pTab->aTmpVec = (float*)vec1MallocZero(nReq * sizeof_f32);
if( pTab->aTmpVec ){
pTab->nTmpVec = nReq;
}else{
pTab->nTmpVec = 0;
rc = SQLITE_NOMEM;
}
}
if( rc==SQLITE_OK
&& pTab->mod.hdr.nElem!=0
&& (u32)pTab->cfg.nElem!=pTab->mod.hdr.nElem
){
vec1VtabError(pTab, "model/data vector size mismatch: %d/%d",
pTab->mod.hdr.nElem, pTab->cfg.nElem
);
vec1UnloadModel(pTab);
rc = SQLITE_ERROR;
}
return rc;
}
/*
** Load the configuration if it is not already loaded. The configuration
** is assumed not to be loaded if cfg.nElem==0.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
** an error occurs, this function may leave an error message in the virtual
** table object.
*/
static int vec1LoadConfigIfRequired(Vec1Tab *pTab){
int rc = SQLITE_OK;
if( pTab->cfg.nElem==0 ){
rc = vec1LoadConfig(pTab);
if( rc!=SQLITE_OK ){
pTab->cfg.nElem = 0;
}
}
return rc;
}
static int vec1InterpretNProbe(Vec1Tab *pTab, double fVal){
int nProbe;
if( fVal>=1.0 ){
nProbe = (int)fVal;
}else{
nProbe = (int)(fVal * pTab->mod.hdr.nBucket);
}
nProbe = MIN(nProbe, (int)pTab->mod.hdr.nBucket);
nProbe = MAX(nProbe, 1);
return nProbe;
}
static int vec1ParseQueryParamCb(
void *pCtx,
const char *zOpt,
int eType,
i64 iVal,
double fVal,
const char *zVal,
char **pzErr
){
Vec1Query *pQuery = (Vec1Query*)pCtx;
const char *azOpt[] = {
"nprobe", /* 0 */
"K", /* 1 */
"streaming", /* 2 */
"nprobe_slack", /* 3 */
0
};
int iOpt = 0;
UNUSED_PARAMETER2(iVal, eType);
UNUSED_PARAMETER2(pzErr, zVal);
int rc = vec1ConfigEnum(&iOpt, zOpt, azOpt, "option", pzErr);
if( rc==SQLITE_OK ){
switch( iOpt ){
case 0: assert( sqlite3_stricmp("nprobe", zOpt)==0 );
if( fVal<=0.0 ){
*pzErr = sqlite3_mprintf(
"vec1: nprobe must be a numeric value greater than 0.0"
);
rc = SQLITE_ERROR;
}
pQuery->nProbe = vec1InterpretNProbe(pQuery->pTab, fVal);
break;
case 1: assert( sqlite3_stricmp("K", zOpt)==0 );
if( eType==SQLITE_INTEGER ){
pQuery->K = iVal;
}else{
pQuery->K = -1;
}
break;
case 2: assert( sqlite3_stricmp("streaming", zOpt)==0 );
pQuery->bStreaming = ((int)fVal) ? 1 : 0;
break;
default: assert( sqlite3_stricmp("nprobe_slack", zOpt)==0 );
pQuery->nProbeSlack = fVal;
break;
}
}
return rc;
}
static int vec1ModelIsFlat(Vec1Model *pMod){
return ((pMod->hdr.flags & VEC1_MODEL_INDEX) && pMod->hdr.nCodebook==0);
}
static const char aHexDigit[] = {
'0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
};
static int vec1DecodeMetaIdx(const char *z){
int iRet = 0;
assert( (z[0]>='0' && z[0]<='9') || (z[0]>='a' && z[0]<='f') );
assert( (z[1]>='0' && z[1]<='9') || (z[1]>='a' && z[1]<='f') );
if( z[0]<='9' ){
iRet = (int)(z[0] - '0');
}else{
iRet = (int)(10 + z[0] - 'a');
}
iRet = iRet << 4;
if( z[1]<='9' ){
iRet |= (int)(z[1] - '0');
}else{
iRet |= (int)(10 + z[1] - 'a');
}
return iRet;
}
/*
** This is invoked when a value of type eType (which is not SQLITE_BLOB)
** has been passed to vec1 as a vector. This is an error. Configure the
** table with an error message and return SQLITE_ERROR.
*/
static int vec1VectorTypeError(Vec1Tab *pTab, int eType){
const char *azType[] = {0, "INTEGER", "FLOAT", "TEXT", "BLOB", "NULL"};
assert( eType!=SQLITE_BLOB && eType>=1 && eType<=5 );
vec1VtabError(pTab,
"vec1: unexpected vector type %s, expected BLOB", azType[eType]
);
return SQLITE_ERROR;
}
/*
** This is invoked when a blob nHave bytes in size has been passed to vec1
** as a vector when a (pTab->cfg.nElem*sizeof_f32) blob was expected.
** Configure the table with an error message and return SQLITE_ERROR.
*/
static int vec1VectorSizeError(Vec1Tab *pTab, int nHave){
assert( pTab->cfg.nElem>0 && (pTab->cfg.nElem*sizeof_f32)!=nHave );
vec1VtabError(pTab,
"vec1: unexpected vector blob size %d bytes, expected %d",
nHave, pTab->cfg.nElem*sizeof_f32
);
return SQLITE_ERROR;
}
static int vec1FilterArraySize(
const char *idxStr,
sqlite3_value **argv,
int *pnFilter
){
int ii;
int nFilter = 0;
int iArg = 1;
int rc = SQLITE_OK;
for(ii=0; idxStr[ii]; ii++){
char c = idxStr[ii];
if( c==VEC1_OP_IN ){
sqlite3_value *pDummy = 0;
sqlite3_value *pList = argv[iArg];
nFilter++;
for(rc=sqlite3_vtab_in_first(pList, &pDummy);
rc==SQLITE_OK;
rc=sqlite3_vtab_in_next(pList, &pDummy)
){
assert( pDummy!=0 );
nFilter++;
}
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
ii += 2;
}else if( c!=VEC1_OP_PARAMS && c!=VEC1_OP_LIMIT ){
ii += 2;
nFilter++;
}
iArg++;
}
*pnFilter = nFilter;
return rc;
}
static void *vec1Dup(int *pRc, const void *p, int n){
void *pRet = 0;
assert( *pRc==SQLITE_OK );
if( p==0 ){
*pRc = SQLITE_NOMEM;
}else{
pRet = vec1MallocZero(n+1);
if( pRet==0 ){
*pRc = SQLITE_NOMEM;
}else if( n>0 ){
memcpy(pRet, p, n);
}
}
return pRet;
}
static void vec1ValueToFilter(
int *pRc,
sqlite3_value *pVal,
Vec1Filter *pFilter
){
pFilter->eType = sqlite3_value_type(pVal);
switch( pFilter->eType ){
case SQLITE_NULL:
break;
case SQLITE_INTEGER:
pFilter->iVal = sqlite3_value_int64(pVal);
break;
case SQLITE_FLOAT:
pFilter->fVal = sqlite3_value_double(pVal);
break;
case SQLITE_TEXT:
pFilter->iVal = sqlite3_value_bytes(pVal);
pFilter->pPtr = (u8*)vec1Dup(
pRc, sqlite3_value_text(pVal), (int)pFilter->iVal
);
break;
default: {
int nByte = sqlite3_value_bytes(pVal);
assert( pFilter->eType==SQLITE_BLOB );
pFilter->iVal = (i64)nByte;
pFilter->pPtr = (u8*)vec1Dup(
pRc, nByte? sqlite3_value_blob(pVal) : (const void*)"", nByte
);
break;
}
}
}
/*
** Return true if value pVal contains an integer - either an SQL integer,
** text that looks like an integer, or a double that can be converted
** to an integer without loss of data.
*/
static int vecIsIntegerValue(sqlite3_value *pVal){
if( sqlite3_value_numeric_type(pVal)==SQLITE_INTEGER ) return 1;
if( sqlite3_value_numeric_type(pVal)==SQLITE_FLOAT ){
double fVal = sqlite3_value_double(pVal);
if( fVal==(double)((i64)fVal) ) return 1;
}
return 0;
}
static int vec1SetupKANNQuery(
Vec1Tab *pTab,
const char *idxStr, /* idxStr passed to xFilter */
sqlite3_value **argv, /* Parameter values passed to xFilter */
Vec1Query **ppOut /* OUT: Query object */
){
const int nElem = pTab->cfg.nElem;
const int nSpace = MAX(pTab->cfg.nElem, pTab->nTmpVec);
int nMax = 0; /* Max possible meta-data filters */
int nVec = sqlite3_value_bytes(argv[0]);
const u8 *aVec = sqlite3_value_blob(argv[0]);
int iArg = 1;
int iIdxStr = 0;
int rc = SQLITE_OK;
Vec1Query *p = 0;
int bTransform = vec1TransformRequired(&pTab->mod);
int nBit = 0;
if( pTab->mod.hdr.flags & VEC1_MODEL_BITQUANT ){
nBit = sizeof_f32 + pTab->mod.hdr.nCodebook;
}
/* Determine the required size of the Vec1Query.aFilter[] array. */
rc = vec1FilterArraySize(idxStr, argv, &nMax);
p = (Vec1Query*)vec1MallocZero(
sizeof(Vec1Query) /* Vec1Query object itself */
+ nMax * sizeof(Vec1Filter) /* Vec1Query.aFilter[] */
+ nSpace * sizeof_f32 /* Vec1Query.aVector[] */
+ bTransform * (nSpace * sizeof_f32) /* aTransform[] */
+ nBit /* aBit[] */
);
if( p==0 ) return SQLITE_NOMEM;
p->aFilter = (Vec1Filter*)&p[1];
p->aVector = (float*)&p->aFilter[nMax];
if( bTransform ){
p->aTransform = &p->aVector[nSpace];
}else{
p->aTransform = p->aVector;
}
if( nBit ){
p->aBit = (u8*)&p->aTransform[nSpace];
}
/* Initialize default query parameters.
**
** Use VEC1_SMALLEST_INT64 to signify that no K value has been supplied
** If somebody specifies this value via a JSON query parameter, they
** may get a slightly inaccurate error message. So be it. */
p->pTab = pTab;
p->K = VEC1_SMALLEST_INT64;
p->nProbe = vec1InterpretNProbe(pTab, pTab->pTabList->nProbeArg);
p->bStreaming = 0;
/* Check the query vector is the right size and copy it into the
** aVector[] buffer.
**
** Omit this check if nElem==0. This occurs when the table has not yet
** received the first vector. So carry on checking the remainder of the
** parameters, but don't worry about checking the input vector or
** copying it into aVector[], as the query will return zero rows in
** any case. */
if( nElem!=0 ){
if( nVec!=(nElem*sizeof_f32) ){
rc = vec1VectorSizeError(pTab, nVec);
}else{
memcpy(p->aVector, aVec, nVec);
}
}
/* Check that the query vector does not contain any NaN or Infinity values */
if( rc==SQLITE_OK ){
rc = vec1VectorIsNonFinite((float*)aVec, (nVec/sizeof_f32));
if( rc!=SQLITE_OK ){
vec1VtabError(pTab, "vec1: non-finite value not allowed in vector");
}
}
/* Check if parameters where supplied via hidden column "arg". If so,
** the first character of idxStr is 'P' - parameters. */
if( rc==SQLITE_OK && idxStr[0]==VEC1_OP_PARAMS ){
sqlite3_value *pVal = argv[iArg++];
if( vecIsIntegerValue(pVal) ){
p->K = sqlite3_value_int64(pVal);
}else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){
assert( pTab->base.zErrMsg==0 );
rc = vec1ParseJsonConfig(pTab->db,
(const char*)sqlite3_value_text(pVal), vec1ParseQueryParamCb,
(void*)p, &pTab->base.zErrMsg
);
}else{
p->K = -1;
}
iIdxStr = 1;
}
/* Process remaining constraints, if any. */
if( rc==SQLITE_OK && idxStr[iIdxStr]!='\0' ){
char c;
while( (c = idxStr[iIdxStr++])!='\0' ){
sqlite3_value *pVal = argv[iArg++];
if( c==VEC1_OP_LIMIT ){
int iVal = sqlite3_value_int(pVal);
if( iVal>=0 && (p->K==VEC1_SMALLEST_INT64 || p->K>0) ){
/* A non-negative LIMIT clause. If no K value was specified, use
** this value as K. Or, if a K value was specified, use the
** minimum of K and this value. */
p->K = ((p->K < 0) ? iVal : MIN(p->K, iVal));
}
}else{
/* Filter on a meta value. The operand character is followed by
** 2 hexadecimal digits encoding the index of the meta column
** on the LHS of the constraint. */
Vec1Filter *pFilter = &p->aFilter[p->nFilter++];
pFilter->vop = c;
pFilter->iMeta = vec1DecodeMetaIdx(&idxStr[iIdxStr]);
iIdxStr += 2;
if( c==VEC1_OP_IN ){
sqlite3_value *pInVal = 0;
assert( pFilter->iVal==0 );
for(rc=sqlite3_vtab_in_first(pVal, &pInVal);
rc==SQLITE_OK;
rc=sqlite3_vtab_in_next(pVal, &pInVal)
){
Vec1Filter *pInFilter = &p->aFilter[p->nFilter++];
vec1ValueToFilter(&rc, pInVal, pInFilter);
if( rc!=SQLITE_OK ) break;
pInFilter->vop = VEC1_OP_EQ;
pInFilter->iMeta = pFilter->iMeta;
pFilter->iVal++;
}
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}else if( c==VEC1_OP_NOTNULL ){
pFilter->eType = SQLITE_BLOB;
pFilter->iVal = 0;
}else if( c==VEC1_OP_ISNULL ){
pFilter->eType = SQLITE_NULL;
pFilter->vop = VEC1_OP_IS;
}else{
vec1ValueToFilter(&rc, pVal, pFilter);
}
}
}
}
assert( rc!=SQLITE_OK || p->nFilter==nMax );
/* Check that a usable K value has been specified. Either explicitly or
** via a visible LIMIT clause. */
if( rc==SQLITE_OK ){
if( p->K==VEC1_SMALLEST_INT64 ){
vec1VtabError(pTab, "vec1: no K value or visible LIMIT clause");
rc = SQLITE_ERROR;
}else if( p->K<0 || p->K>VEC1_MAX_K ){
vec1VtabError(pTab,
"vec1: K must be an integer value between 0 and %d", VEC1_MAX_K
);
rc = SQLITE_ERROR;
}
}
/* If an error occurred, free any dynamic allocations and zero the output
** structure before returning. Otherwise, sort the aFilter[] array by
** iMeta value. */
if( rc!=SQLITE_OK ){
vec1QueryFree(p);
p = 0;
}else if( 0 ){
/* Sort aFilter[] by Vec1Filter.iMeta value. The sort must be stable,
** in order to avoid disrupting the arguments to a VEC1_OP_IN filter. */
int i1, i2;
for(i1=0; i1<p->nFilter; i1++){
Vec1Filter tmp = p->aFilter[i1];
for(i2=i1-1; i2>=0 && p->aFilter[i2].iMeta>tmp.iMeta; i2--){
p->aFilter[i2+1] = p->aFilter[i2];
}
p->aFilter[i2+1] = tmp;
}
#ifndef NDEBUG
/* Assert() that the sort worked. */
for(i1=1; i1<p->nFilter; i1++){
assert( p->aFilter[i1-1].iMeta<=p->aFilter[i1].iMeta );
}
#endif
}
*ppOut = p;
return rc;
}
/*
** This function is called in index:"none" mode.
*/
static int vec1QueryToSql(Vec1Csr *pCsr){
Vec1Tab *pTab = (Vec1Tab*)(pCsr->base.pVtab);
Vec1Query *pQuery = pCsr->pQuery;
int ii;
int rc = SQLITE_OK;
char *zWhere = 0;
int iNextVar = 1;
sqlite3_str *pWhere = sqlite3_str_new(pTab->db);
const char *zAnd = "";
for(ii=0; ii<pQuery->nFilter; ii++){
Vec1Filter *p = &pQuery->aFilter[ii];
int bRhs = 1;
const char *zOp = "";
if( p->vop==VEC1_OP_IN ){
int jj;
sqlite3_str_appendf(pWhere, "%sc%d IN(?%d", zAnd, p->iMeta, iNextVar++);
for(jj=2; jj<=p->iVal; jj++){
sqlite3_str_appendf(pWhere, ", ?%d", iNextVar++);
}
ii += (int)p->iVal;
sqlite3_str_appendf(pWhere, ")");
}else{
switch( p->vop ){
case VEC1_OP_EQ: zOp = "="; break;
case VEC1_OP_LT: zOp = "<"; break;
case VEC1_OP_GT: zOp = ">"; break;
case VEC1_OP_LE: zOp = "<="; break;
case VEC1_OP_GE: zOp = ">="; break;
case VEC1_OP_IS: zOp = "IS"; break;
default:
assert( p->vop==VEC1_OP_NOTNULL );
zOp = "IS NOT NULL";
bRhs = 0;
break;
}
sqlite3_str_appendf(pWhere, "%sc%d %s", zAnd, p->iMeta, zOp);
if( bRhs ){
sqlite3_str_appendf(pWhere, " ?%d", iNextVar++);
}
}
zAnd = " AND ";
}
rc = sqlite3_str_errcode(pWhere);
zWhere = sqlite3_str_finish(pWhere);
if( rc==SQLITE_OK ){
const char *zDistance = "vec1_l2_distance";
if( pTab->mod.hdr.eDistance==VEC1_DISTANCE_COS ){
zDistance = "vec1_cos_distance";
}
rc = vec1PrepareSql(pTab, &pCsr->pStmt,
"SELECT *, %s(?%d, vector) AS d FROM %Q.'%q_base'%s%s "
"ORDER BY d LIMIT %lld",
zDistance, iNextVar,
pTab->zDb, pTab->zName,
zWhere ? " WHERE " : "", zWhere,
pQuery->K
);
}
if( rc==SQLITE_OK ){
int n = pTab->cfg.nElem * sizeof_f32;
rc = sqlite3_bind_blob(
pCsr->pStmt, iNextVar, pQuery->aVector, n, SQLITE_TRANSIENT
);
iNextVar = 1;
for(ii=0; rc==SQLITE_OK && ii<pQuery->nFilter; ii++){
Vec1Filter *p = &pQuery->aFilter[ii];
if( p->vop!=VEC1_OP_IN && p->vop!=VEC1_OP_NOTNULL ){
switch( p->eType ){
case SQLITE_NULL:
break;
case SQLITE_INTEGER:
sqlite3_bind_int64(pCsr->pStmt, iNextVar, p->iVal);
break;
case SQLITE_FLOAT:
sqlite3_bind_double(pCsr->pStmt, iNextVar, p->fVal);
break;
case SQLITE_TEXT:
rc = sqlite3_bind_text(pCsr->pStmt,
iNextVar, (const char*)p->pPtr, (int)p->iVal, SQLITE_TRANSIENT
);
break;
default:
assert( p->eType==SQLITE_BLOB );
rc = sqlite3_bind_blob(
pCsr->pStmt, iNextVar, p->pPtr, (int)p->iVal, SQLITE_TRANSIENT
);
break;
}
iNextVar++;
}
}
}
vec1QueryFree(pQuery);
pCsr->pQuery = 0;
pCsr->bSqlNNQuery = 1;
sqlite3_free(zWhere);
return rc;
}
/*
** There are three basic query plans supported by xBestIndex:
**
** 0: Full table scan.
** 1: Lookup by rowid.
** 2: K-nearest-neighbours search.
*/
static int vec1FilterMethod(
sqlite3_vtab_cursor *cur,
int idxNum,
const char *idxStr,
int argc,
sqlite3_value **argv
){
Vec1Csr *pCsr = (Vec1Csr *)cur;
Vec1Tab *pTab = (Vec1Tab*)(cur->pVtab);
int rc = SQLITE_OK;
UNUSED_PARAMETER(argc);
assert( ((idxNum==0 || idxNum==1) && argc==0)
|| ((idxNum==2 || idxNum==3) && argc==1)
|| (idxNum==5 && argc>=1)
);
vec1ResetCsr(pCsr);
if( pCsr->bLoadConfig==0 && (idxNum & 0x01) ){
rc = vec1LoadConfig(pTab);
if( rc!=SQLITE_OK ) return rc;
pCsr->bLoadConfig = 1;
}
if( idxNum==0 || idxNum==1 ){
/* Full-table scan. */
rc = vec1PrepareSql(pTab, &pCsr->pStmt,
"SELECT * FROM %Q.'%q_base'", pTab->zDb, pTab->zName
);
}else if( idxNum==2 || idxNum==3 ){
/* Lookup by rowid */
rc = vec1PrepareSql(pTab, &pCsr->pStmt,
"SELECT * FROM %Q.'%q_base' WHERE rowid=?",
pTab->zDb, pTab->zName
);
if( rc==SQLITE_OK ){
sqlite3_bind_value(pCsr->pStmt, 1, argv[0]);
}
}else{
/* ANN query */
Vec1Query *pQuery = 0;
assert( idxNum==5 );
VEC1_QINSTR_START(pTab, VEC1_QINSTR_TOTAL);
/* Parse the query parameters. */
rc = vec1SetupKANNQuery(pTab, idxStr, argv, &pQuery);
pCsr->pQuery = pQuery;
if( rc==SQLITE_OK ){
if( (pTab->mod.hdr.flags & VEC1_MODEL_INDEX)==0 ){
rc = vec1QueryToSql(pCsr);
assert( pCsr->pQuery==0 );
assert( pCsr->bSqlNNQuery );
}else{
rc = vec1PrepareSql(pTab, &pCsr->pStmt,
"SELECT * FROM %Q.'%q_base' WHERE id=?",
pTab->zDb, pTab->zName
);
if( rc==SQLITE_OK ){
rc = vec1DoKANNQuery(pCsr);
pCsr->iCurrentRes = -1;
}
}
}
VEC1_QINSTR_STOP(pTab, VEC1_QINSTR_TOTAL);
}
if( rc==SQLITE_OK ){
rc = vec1NextMethod(cur);
}
return rc;
}
static int vec1EofMethod(sqlite3_vtab_cursor *cur){
Vec1Csr *pCur = (Vec1Csr *)cur;
return (pCur->pStmt==0);
}
static int vec1SeekCsr(Vec1Csr *pCsr){
int rc = SQLITE_OK;
if( pCsr->pQuery && pCsr->bSeek==0 ){
i64 iRowid = pCsr->pQuery->heap.aRes[pCsr->iCurrentRes].iRowid;
sqlite3_bind_int64(pCsr->pStmt, 1, iRowid);
if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
rc = sqlite3_finalize(pCsr->pStmt);
if( rc==SQLITE_OK ){
rc = VEC1_CORRUPT;
}
pCsr->pStmt = 0;
}
pCsr->bSeek = 1;
}
return rc;
}
typedef struct Vec1RowidLocation Vec1RowidLocation;
struct Vec1RowidLocation {
int iEntry;
int nEntry;
int nTombstone;
int szRowid;
Vec1ListBuilder *pList;
sqlite3_blob *pBlob;
i64 iIdx;
};
static int vec1FindByRowid(
Vec1Tab *pTab,
int iBucket, /* Row belongs to this bucket */
int bWrite, /* Use writable blob-handle */
i64 iRowid, /* Rowid to locate */
Vec1RowidLocation *pOut /* OUT: Result */
){
int rc = SQLITE_OK;
Vec1Buffer buf = {0, 0, 0};
sqlite3_stmt *pSearch = 0; /* For looping through index entries */
sqlite3_blob *pBlob = 0;
assert( (pTab->mod.hdr.flags & VEC1_MODEL_INDEX) );
memset(pOut, 0, sizeof(*pOut));
/* Check to see if the index entry for this vector is currently
** in main memory. */
if( pTab->pWriter ){
Vec1ListBuilder *pBld = &pTab->pWriter->aBld[iBucket];
int ii;
for(ii=0; ii<pBld->bufRowid.n; ii+=pBld->szRowid){
i64 iRead;
if( pBld->szRowid==4 ){
iRead = vec1GetU32(&pBld->bufRowid.a[ii]);
}else{
iRead = vec1GetU64(&pBld->bufRowid.a[ii]);
}
if( iRead==iRowid ){
pOut->iEntry = (ii / pBld->szRowid);
pOut->nEntry = (pBld->bufRowid.n / pBld->szRowid);
pOut->nTombstone = pBld->nTombstone;
pOut->szRowid = pBld->szRowid;
pOut->pList = pBld;
return SQLITE_OK;
}
}
}
rc = vec1GetSql(pTab, VEC1_SQL_SEARCH_IDX, &pSearch);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pSearch, 1, iBucket);
sqlite3_bind_int64(pSearch, 2, iRowid);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSearch) ){
int nEntry = 0;
u32 nTombstone = 0;
u32 flags = 0;
int szRowid = 4;
i64 iIdx = sqlite3_column_int64(pSearch, 0);
rc = sqlite3_blob_open(
pTab->db, pTab->zDb, pTab->zIdxTbl, "val", iIdx, bWrite, &pBlob
);
if( rc==SQLITE_OK ){
u8 a[VEC1_LIST_SZHDR];
rc = sqlite3_blob_read(pBlob, a, VEC1_LIST_SZHDR, 0);
flags = vec1GetU32(&a[0]);
nEntry = (int)vec1GetU32(&a[4]);
nTombstone = vec1GetU32(&a[8]);
if( flags & VEC1_LIST_64BIT ) szRowid = 8;
}
/* Now scan through the array of rowids at the start of the %_idx entry
** to find the rowid to be deleted. But, do not do this if the rowid
** happens to have the same value as the tombstone for this %_idx entry. */
if( (szRowid==4 && iRowid!=VEC1_TOMBSTONE_32)
|| (szRowid==8 && iRowid!=VEC1_TOMBSTONE_64)
){
if( rc==SQLITE_OK ){
buf.n = 0;
rc = vec1BufferGrow(&buf, VEC1_LIST_SZHDR+szRowid*nEntry);
}
if( rc==SQLITE_OK ){
rc = sqlite3_blob_read(
pBlob, &buf.a[VEC1_LIST_SZHDR], nEntry*szRowid, VEC1_LIST_SZHDR
);
}
if( rc==SQLITE_OK ){
int ii;
for(ii=0; ii<nEntry; ii++){
i64 iRead = 0;
if( szRowid==4 ){
iRead = vec1GetU32(&buf.a[VEC1_LIST_SZHDR + ii*szRowid]);
}else{
iRead = vec1GetU64(&buf.a[VEC1_LIST_SZHDR + ii*szRowid]);
}
if( iRead==iRowid ){
pOut->iEntry = ii;
pOut->nEntry = nEntry;
pOut->nTombstone = nTombstone;
pOut->szRowid = szRowid;
pOut->pBlob = pBlob;
pOut->iIdx = iIdx;
pBlob = 0;
goto findbyrowid_done;
}
}
}
}
(void)sqlite3_blob_close(pBlob);
}
findbyrowid_done:
vec1StmtReset(&rc, pSearch);
vec1BufferFree(&buf);
return rc;
}
/*
** Obtain the full vector associated with the current row of cursor pCsr.
*/
static int vec1GetVector(
Vec1Csr *pCsr,
const u8 **paVec, /* OUT: Pointer to vector buffer */
int *pnVec, /* OUT: Size of (*paVec) in bytes */
u8 **pFree /* OUT: Buffer for caller to free (or NULL) */
){
int rc = SQLITE_OK;
Vec1Tab *pTab = (Vec1Tab*)(pCsr->base.pVtab);
rc = vec1SeekCsr(pCsr);
if( rc==SQLITE_OK ){
if( vec1ModelIsFlat(&pTab->mod) ){
i64 iRowid = sqlite3_column_int64(pCsr->pStmt, 0);
int iBucket = sqlite3_column_int(pCsr->pStmt, 1);
Vec1RowidLocation loc;
rc = vec1FindByRowid(pTab, iBucket, 0, iRowid, &loc);
if( rc==SQLITE_OK ){
int nVec = sizeof_f32 * pTab->cfg.nElem;
u8 *aVec = (u8*)sqlite3_malloc(nVec);
if( aVec==0 ){
rc = SQLITE_NOMEM;
}else if( loc.pList ){
memcpy(aVec, &loc.pList->bufData.a[nVec * loc.iEntry], nVec);
}else if( loc.pBlob ){
rc = sqlite3_blob_read(loc.pBlob, aVec, nVec,
VEC1_LIST_SZHDR + (loc.nEntry*loc.szRowid) + (loc.iEntry * nVec)
);
}else{
sqlite3_free(aVec);
nVec = 0;
aVec = 0;
}
if( rc!=SQLITE_OK ){
sqlite3_free(aVec);
*paVec = 0;
*pnVec = 0;
*pFree = 0;
}else{
*paVec = *pFree = aVec;
*pnVec = nVec;
}
(void)sqlite3_blob_close(loc.pBlob);
}
}else{
*paVec = sqlite3_column_blob(pCsr->pStmt, 1);
*pnVec = sqlite3_column_bytes(pCsr->pStmt, 1);
*pFree = 0;
}
}
return rc;
}
/*
** This function quantizes vector aIn[], then dequantizes it and finds
** the squared L2 distance between the original and the round-tripped
** vector. If successful, output parameter (*prError) is set to the
** squared distance value and SQLITE_OK is returned. Or, if an error
** occurs, an SQLite error code (SQLITE_NOMEM) is returned.
*/
static int vec1QuantizeReconstructionError2(
Vec1Model *pMod, /* Model to use for quantization */
const float *aIn, /* Input vector */
char **pzEnc,
double *prError /* OUT: Round trip error */
){
const int nElem = (int)pMod->hdr.nElem;
float *aRound = 0;
u8 *aCode = 0;
int nCode = vec1EncodedVectorSize(pMod);
char *zEnc = 0;
int iEnc = 0;
int ii;
zEnc = (char*)vec1MallocZero(
nElem * 8 +
nElem * sizeof_f32 + /* aRound[] */
nCode /* aCode[] */
);
if( zEnc==0 ){
return SQLITE_NOMEM;
}
aRound = (float*)&zEnc[nElem * 8];
aCode = (u8*)&aRound[nElem];
vec1EncodeVector(pMod, aIn, aCode);
vec1DecodeVector(pMod, aCode, aRound);
*prError = vec1L2Dist(aIn, aRound, nElem);
zEnc[iEnc++] = '[';
for(ii=0; ii<nCode; ii++){
if( ii!=0 ) zEnc[iEnc++] = ',';
if( aCode[ii]>=100 ) zEnc[iEnc++] = (aCode[ii] / 100) + '0';
if( aCode[ii]>=10 ) zEnc[iEnc++] = ((aCode[ii] / 10) % 10) + '0';
zEnc[iEnc++] = (aCode[ii] % 10) + '0';
}
zEnc[iEnc++] = ']';
*pzEnc = zEnc;
return SQLITE_OK;
}
static void vec1DistanceStats(sqlite3_context *ctx, Vec1Csr *pCsr){
int rc = SQLITE_OK;
if( pCsr->zDistance==0 ){
const float *aVec = 0;
int nVec = 0;
u8 *pFree = 0;
rc = vec1GetVector(pCsr, (const u8**)&aVec, &nVec, &pFree);
if( rc==SQLITE_OK && aVec ){
Vec1Tab *pTab = (Vec1Tab*)(pCsr->base.pVtab);
int nElem = pTab->cfg.nElem;
Vec1Model *pMod = &pTab->mod;
int iBucket = 0;
double fCoarseError = 0.0;
double fReconError = 0.0;
double fNorm = 0.0;
const float *aTransform = 0;
char *zEnc = 0;
if( pMod->hdr.nBucket>1 || pMod->hdr.nCodebook>0 ){
aTransform = vec1TransformInputVector(pMod, pTab->aTmpVec, aVec);
fNorm = vec1VectorNorm2(aTransform, nElem);
}
/* Find bucket and coarse error if applicable. */
if( pMod->hdr.nBucket>1 ){
iBucket = vec1BestMatchSimple(
pMod->aCentroid, pMod->hdr.nBucket, aVec, nElem, &fCoarseError
);
if( aTransform!=pTab->aTmpVec ){
memcpy(pTab->aTmpVec, aTransform, nElem*sizeof_f32);
aTransform = pTab->aTmpVec;
}
if( pTab->mod.hdr.flags & VEC1_MODEL_RESIDUAL ){
vec1SubInPlace(pTab->aTmpVec, &pMod->aCentroid[iBucket*nElem], nElem);
}
}
/* Find reconstruction error if applicable. */
if( pMod->hdr.nCodebook>0 ){
rc = vec1QuantizeReconstructionError2(
pMod, aTransform, &zEnc, &fReconError
);
}
pCsr->zDistance = vec1MPrintf(&rc,
"{bucket:%d, coarse_error:%f, reconstruction_error:%f, code: %s}",
iBucket,
fNorm==0.0 ? 0.0 : sqrt(fCoarseError/fNorm),
fNorm==0.0 ? 0.0 : sqrt(fReconError/fNorm),
zEnc ? zEnc : "null"
);
sqlite3_free(zEnc);
}
sqlite3_free(pFree);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(ctx, rc);
}else{
sqlite3_result_text(ctx, pCsr->zDistance, -1, SQLITE_TRANSIENT);
}
}
static int vec1ColumnMethod(
sqlite3_vtab_cursor *cur,
sqlite3_context *ctx,
int iCol
){
Vec1Csr *pCsr = (Vec1Csr *)cur;
int rc = SQLITE_OK;
if( iCol==VEC1_COLUMN_DISTANCE ){
if( pCsr->pQuery ){
double fDist = pCsr->pQuery->heap.aRes[pCsr->iCurrentRes].fDist;
sqlite3_result_double(ctx, fDist);
}else if( pCsr->bSqlNNQuery ){
int ii = sqlite3_column_count(pCsr->pStmt)-1;
sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, ii));
}else{
vec1DistanceStats(ctx, pCsr);
}
}else if( iCol==VEC1_COLUMN_VECTOR ){
const u8 *aVec = 0;
int nVec = 0;
u8 *pFree = 0;
rc = vec1GetVector(pCsr, &aVec, &nVec, &pFree);
if( rc==SQLITE_OK ){
sqlite3_result_blob(ctx, aVec, nVec, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(ctx, rc);
}
sqlite3_free(pFree);
}else if( iCol>VEC1_COLUMN_VECTOR ){
rc = vec1SeekCsr(pCsr);
if( rc==SQLITE_OK ){
sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, iCol-2));
}
}
return rc;
}
static int vec1Rowid(
sqlite3_vtab_cursor *cur,
sqlite3_int64 *pRowid
){
Vec1Csr *pCsr = (Vec1Csr *)cur;
if( pCsr->pQuery ){
*pRowid = pCsr->pQuery->heap.aRes[pCsr->iCurrentRes].iRowid;
}else{
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
return SQLITE_OK;
}
/*
** Return true if constraint iCons of pInfo uses the "binary" collation
** sequence. Or false it it uses anything else.
*/
static int vec1IsBinary(sqlite3_index_info *pInfo, int iCons){
return 0==sqlite3_stricmp("binary", sqlite3_vtab_collation(pInfo, iCons));
}
/*
** xBestIndex implementation. Three plans are supported:
**
** + Full scan. (idxNum==0 or idxNum=1)
** + Lookup by rowid . (idxNum==2 or idxNum=3)
** + KANN search. (idxNum==5)
**
** For KANN search, meta-data constraints are supported. They are encoded
** in idxStr. Specifically, for each handled constraint, there is one
** character to identify the operation, usually followed by one or more
** characters to identify the column the constraint applies to.
**
** idxStr format:
*/
static int vec1BestIndexMethod(
sqlite3_vtab *pVtab,
sqlite3_index_info *pIdxInfo
){
struct Vec1IdxOp {
int idxop;
char vec1op;
} aOp[] = {
{ SQLITE_INDEX_CONSTRAINT_EQ, VEC1_OP_EQ },
{ SQLITE_INDEX_CONSTRAINT_LT, VEC1_OP_LT },
{ SQLITE_INDEX_CONSTRAINT_GT, VEC1_OP_GT },
{ SQLITE_INDEX_CONSTRAINT_LE, VEC1_OP_LE },
{ SQLITE_INDEX_CONSTRAINT_GE, VEC1_OP_GE },
{ SQLITE_INDEX_CONSTRAINT_IS, VEC1_OP_IS },
{ SQLITE_INDEX_CONSTRAINT_ISNULL, VEC1_OP_ISNULL },
{ SQLITE_INDEX_CONSTRAINT_ISNOTNULL, VEC1_OP_NOTNULL },
};
const int bDistance = (
pIdxInfo->colUsed & ((1<<VEC1_COLUMN_DISTANCE)|(1<<VEC1_COLUMN_VECTOR))
);
int iCmd = -1;
int iArg = -1;
int iRowid = -1;
int ii;
UNUSED_PARAMETER(pVtab);
/* First pass. Look for == on VEC1_COLUMN_CMD and VEC1_COLUMN_ARG.
** Or an == on the rowid column. */
for(ii=0; ii<pIdxInfo->nConstraint; ii++){
struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii];
if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
if( p->usable ){
if( p->iColumn==VEC1_COLUMN_CMD ) iCmd = ii;
else if( p->iColumn==VEC1_COLUMN_ARG ) iArg = ii;
else if( p->iColumn<0 ) iRowid = ii;
}else{
if( p->iColumn==VEC1_COLUMN_CMD || p->iColumn==VEC1_COLUMN_ARG ){
return SQLITE_CONSTRAINT;
}
}
}
}
if( iCmd>=0 ){
int iLimit = -1;
char *idxStr = vec1MallocZero((pIdxInfo->nConstraint*8)+1);
int iStr = 0;
int argvIndex = 0;
if( idxStr==0 ) return SQLITE_NOMEM;
pIdxInfo->idxNum = 5;
pIdxInfo->needToFreeIdxStr = 1;
pIdxInfo->idxStr = idxStr;
pIdxInfo->aConstraintUsage[iCmd].argvIndex = (++argvIndex);
pIdxInfo->aConstraintUsage[iCmd].omit = 1;
if( iArg>=0 ){
pIdxInfo->aConstraintUsage[iArg].argvIndex = (++argvIndex);
pIdxInfo->aConstraintUsage[iArg].omit = 1;
idxStr[iStr++] = VEC1_OP_PARAMS;
}
/* Meta-data constraints */
for(ii=0; ii<pIdxInfo->nConstraint; ii++){
struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii];
if( p->usable==0 ) continue;
if( p->iColumn>VEC1_COLUMN_VECTOR ){
int jj;
if( 0==vec1IsBinary(pIdxInfo, ii) ) continue;
for(jj=0; jj<size_of_array(aOp); jj++){
if( p->op==aOp[jj].idxop ){
int iMeta = (p->iColumn - 1 - VEC1_COLUMN_VECTOR);
if( jj==0 && sqlite3_vtab_in(pIdxInfo, ii, -1) ){
idxStr[iStr++] = VEC1_OP_IN;
sqlite3_vtab_in(pIdxInfo, ii, 1);
}else{
idxStr[iStr++] = aOp[jj].vec1op;
}
idxStr[iStr++] = aHexDigit[iMeta>>4];
idxStr[iStr++] = aHexDigit[iMeta&0xF];
pIdxInfo->aConstraintUsage[ii].argvIndex = (++argvIndex);
pIdxInfo->aConstraintUsage[ii].omit = 1;
break;
}
}
}
if( p->op==SQLITE_INDEX_CONSTRAINT_LIMIT ){
iLimit = ii;
}
}
if( iLimit>=0
&& pIdxInfo->nOrderBy==0
&& argvIndex==(pIdxInfo->nConstraint-1)
){
idxStr[iStr++] = VEC1_OP_LIMIT;
pIdxInfo->aConstraintUsage[iLimit].argvIndex = (++argvIndex);
pIdxInfo->aConstraintUsage[iLimit].omit = 1;
}
}else if( iRowid>=0 ){
pIdxInfo->idxNum = (bDistance ? 3 : 2);
pIdxInfo->aConstraintUsage[iRowid].argvIndex = 1;
pIdxInfo->aConstraintUsage[iRowid].omit = 1;
pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
pIdxInfo->estimatedCost = 10.0;
pIdxInfo->estimatedRows = 1;
}else if( bDistance ){
pIdxInfo->idxNum = 1;
}
return SQLITE_OK;
}
static void vec1ListBuilderInitMetaArray(Vec1ListBuilder *p){
const u32 flags = VEC1_META_1BYTEINT|VEC1_META_4BYTEINT|VEC1_META_REAL;
int ii;
for(ii=0; ii<p->pTab->nMeta; ii++){
p->aMeta[ii].flags = flags;
p->aMeta[ii].format = VEC1_META_GENERIC;
p->aMeta[ii].buf.n = VEC1_META_SZHDR;
}
}
static int vec1ListBuilderInit(Vec1ListBuilder *p, Vec1Tab *pTab, int iBucket){
int rc = SQLITE_OK;
p->pTab = pTab;
p->iBucket = iBucket;
p->szRowid = 4;
p->bSorted = 1;
if( pTab->mod.hdr.nCodebook ){
if( pTab->mod.hdr.flags & VEC1_MODEL_BITQUANT ){
p->eBlocked = VEC1_BLOCKED_BIN;
}else{
p->eBlocked = VEC1_BLOCKED_PQ;
}
p->nVectorSize = vec1EncodedVectorSize(&pTab->mod);
}else{
p->nVectorSize = pTab->cfg.nElem * sizeof_f32;
p->eBlocked = VEC1_BLOCKED_NONE;
}
if( pTab->nMeta>0 ){
int nMeta = pTab->nMeta;
p->aMeta = (Vec1MetaBuilder*)vec1MallocZero(sizeof(Vec1MetaBuilder)*nMeta);
if( p->aMeta==0 ){
rc = SQLITE_NOMEM;
}else{
vec1ListBuilderInitMetaArray(p);
}
}
return rc;
}
static int vec1WriteMetaBlob(
Vec1Tab *pTab, /* Vec1 virtual table */
Vec1Buffer *pBuf, /* Buffer to write from */
i64 iId, /* Id of %_idx row */
int iMeta /* Index of meta-column */
){
sqlite3_stmt *pStmt = 0;
int rc = vec1GetSql(pTab, VEC1_SQL_WRITE_META, &pStmt);
assert( iMeta>=0 && iMeta<VEC1_MAX_META_COLUMNS );
if( rc==SQLITE_OK ){
i64 iMetaId = (iId << VEC1_META_COLUMN_BITS) + iMeta;
sqlite3_bind_int64(pStmt, 1, iMetaId);
rc = sqlite3_bind_blob(pStmt, 2, pBuf->a, pBuf->n, SQLITE_STATIC);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
sqlite3_bind_null(pStmt, 2);
}
return rc;
}
static int vec1WriteMeta(
Vec1Tab *pTab, /* Vec1 virtual table */
Vec1MetaBuilder *p, /* Buffer to write from */
int nEntry,
i64 iId, /* Id of %_idx row */
int iMeta /* Index of meta-column */
){
u32 f = 0;
if( (p->format & VEC1_META_GENERIC)
&& (p->flags & (VEC1_META_1BYTEINT|VEC1_META_4BYTEINT|VEC1_META_REAL))
){
int iIn = VEC1_META_SZHDR;
int iOut = VEC1_META_SZHDR;
Vec1MetaValue val = {0,0,0,0};
Vec1Buffer buf = {0, 0, 0};
int rc = SQLITE_OK;
int szElem;
i64 iNull;
if( p->flags & VEC1_META_1BYTEINT ){
szElem = 1;
iNull = VEC1_META_1BYTENULL;
p->format = VEC1_META_1BYTEINT;
}else if( p->flags & VEC1_META_4BYTEINT ){
szElem = sizeof_u32;
iNull = VEC1_META_4BYTENULL;
p->format = VEC1_META_4BYTEINT;
}else{
szElem = sizeof_f64;
iNull = VEC1_META_REALNULL;
p->format = VEC1_META_REAL;
}
rc = vec1BufferSize(&buf, VEC1_META_SZHDR + nEntry * szElem);
if( rc!=SQLITE_OK ) return rc;
while( iIn<p->buf.n ){
rc = vec1MetaValueRead(&p->buf, &iIn, &val);
if( rc!=SQLITE_OK ) break;
if( val.eType==SQLITE_NULL ){
val.iVal = iNull;
}else if( p->format==VEC1_META_REAL ){
if( val.eType==SQLITE_INTEGER ){
val.fVal = (double)val.iVal;
}
memcpy(&val.iVal, &val.fVal, sizeof_f64);
}
if( szElem==1 ){
buf.a[iOut] = (val.iVal & 0xFF);
}else if( szElem==sizeof_u32 ){
vec1PutU32(&buf.a[iOut], (u32)val.iVal);
}else{
vec1PutU64(&buf.a[iOut], (u64)val.iVal);
}
iOut += szElem;
}
buf.n = iOut;
SWAP(Vec1Buffer, p->buf, buf);
vec1BufferFree(&buf);
}
f = (p->format | (p->flags & VEC1_META_HASNULL));
vec1PutU32(&p->buf.a[0], f);
vec1PutU32(&p->buf.a[4], nEntry);
return vec1WriteMetaBlob(pTab, &p->buf, iId, iMeta);
}
/*
** A serialized meta-value is stored in buffer aBuf. Return the size in
** bytes of the meta value.
*/
static i64 vec1MetaValueSize(const u8 *aBuf){
switch( aBuf[0] ){
case 0: return 1; /* NULL */
case 1: return 2; /* 1-byte integer */
case 2: return 5; /* 4-byte float */
case 3: return 9; /* 8-byte integer */
case 4: return 9; /* 8-byte real */
default: {
/* Either a blob or text field. If the size of the value in bytes
** is N, then the varint value V is:
**
** text (odd V values) : V = (2N + 5) -> N = (V - 5) / 2
** blob (even V values): V = (2N + 6) -> N = (V - 6) / 2
*/
i64 V = 0;
int n = vec1GetVarint(aBuf, (u64*)&V);
return (((V-5)/2) & (i64)0xFFFFFFFF) + n;
}
}
}
static int vec1ListBuilderLoad(
Vec1ListBuilder *p,
i64 iId,
i64 iFirst,
i64 iLast,
const u8 *aVal, int nVal
){
Vec1Tab *pTab = p->pTab;
int nEntry = 0;
int flags = 0;
int rc = SQLITE_OK;
int nRowid = 0;
int nData = 0;
int ii; /* Iterator variable */
assert( p->bufRowid.n==0 );
assert( p->szRowid==4 );
if( vec1CheckIdxSize(pTab, aVal, nVal) ){
return VEC1_CORRUPT;
}
flags = vec1GetU32(&aVal[0]);
nEntry = vec1GetU32(&aVal[4]);
p->nTombstone = vec1GetU32(&aVal[8]);
if( flags & VEC1_LIST_64BIT ) p->szRowid = 8;
/* Set Vec1ListBuilder.eBlocked */
if( pTab->mod.hdr.flags & VEC1_MODEL_BITQUANT ){
p->eBlocked = VEC1_BLOCKED_BIN;
}else if( pTab->mod.hdr.nCodebook>0 ){
p->eBlocked = VEC1_BLOCKED_PQ;
}else{
p->eBlocked = VEC1_BLOCKED_NONE;
}
p->bSorted = ((flags & VEC1_LIST_SORTED)!=0);
p->iFirst = iFirst;
p->iLast = iLast;
p->iId = iId;
nRowid = nEntry * p->szRowid;
nData = nVal - VEC1_LIST_SZHDR - nRowid;
rc = vec1BufferGrow(&p->bufRowid, nRowid);
if( rc==SQLITE_OK ){
memcpy(p->bufRowid.a, &aVal[VEC1_LIST_SZHDR], nRowid);
p->bufRowid.n = nRowid;
rc = vec1BufferGrow(&p->bufData, nData);
}
if( rc==SQLITE_OK ){
memcpy(p->bufData.a, &aVal[VEC1_LIST_SZHDR+nRowid], nData);
p->bufData.n = nData;
}
/* Load the array for each meta-data column from the %_meta table */
for(ii=0; rc==SQLITE_OK && ii<pTab->nMeta; ii++){
Vec1MetaBuilder *pMeta = &p->aMeta[ii];
rc = vec1ReadMeta(pTab, nEntry, &pMeta->buf, iId, ii);
if( rc==SQLITE_OK ){
u32 f = vec1GetU32(pMeta->buf.a);
pMeta->format = (f & VEC1_META_TYPEMASK);
pMeta->flags = f;
}
}
return rc;
}
static int vec1ListBuilderFlush(Vec1ListBuilder *p){
int rc = SQLITE_OK;
if( p->bufRowid.n>0 ){
int nEntry = p->bufRowid.n / p->szRowid;
i64 iId = 0;
int ii;
sqlite3_stmt *pStmt = 0;
u8 *aBlob = 0;
int nBlob = 0;
nBlob = VEC1_LIST_SZHDR + p->bufRowid.n + p->bufData.n;
aBlob = (u8*)sqlite3_malloc(nBlob);
if( aBlob==0 ){
rc = SQLITE_NOMEM;
}else{
int flags = 0;
if( p->bSorted ) flags |= VEC1_LIST_SORTED;
if( p->szRowid==8 ) flags |= VEC1_LIST_64BIT;
vec1PutU32(&aBlob[0], flags);
vec1PutU32(&aBlob[4], nEntry);
vec1PutU32(&aBlob[8], p->nTombstone);
memcpy(&aBlob[VEC1_LIST_SZHDR], p->bufRowid.a, p->bufRowid.n);
memcpy(&aBlob[VEC1_LIST_SZHDR+p->bufRowid.n], p->bufData.a, p->bufData.n);
if( p->iId>0 ){
/* If there is an existing entry, remove it */
rc = vec1GetSql(p->pTab, VEC1_SQL_DELETE_FROM_IDX, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pStmt, 1, p->iId);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
}
}
if( rc==SQLITE_OK ){
rc = vec1GetSql(p->pTab, VEC1_SQL_INSERT_IDX, &pStmt);
}
if( rc==SQLITE_OK ){
if( p->iId==0 ){
sqlite3_bind_null(pStmt, 1);
}else{
sqlite3_bind_int64(pStmt, 1, p->iId);
}
sqlite3_bind_int(pStmt, 2, p->iBucket);
sqlite3_bind_int64(pStmt, 3, p->iFirst);
sqlite3_bind_int64(pStmt, 4, p->iLast);
rc = sqlite3_bind_blob(pStmt, 5, aBlob, nBlob, SQLITE_STATIC);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
if( rc==SQLITE_OK ){
iId = sqlite3_last_insert_rowid(p->pTab->db);
}
}
sqlite3_free(aBlob);
/* If an error occurred within the sqlite3_step() call above, then the
** transaction may have been rolled back and (*p) freed. So take
** care not to access it again if rc!=SQLITE_OK. */
if( rc==SQLITE_OK ){
p->bufRowid.n = 0;
p->bufData.n = 0;
p->iId = 0;
p->iId = 0;
}
}
for(ii=0; rc==SQLITE_OK && ii<p->pTab->nMeta; ii++){
rc = vec1WriteMeta(p->pTab, &p->aMeta[ii], nEntry, iId, ii);
if( rc==SQLITE_OK ) p->aMeta[ii].buf.n = VEC1_META_SZHDR;
}
/* Same reasoning as above - if rc!=SQLITE_OK, then the transaction
** may have been rolled back and (*p) freed already. */
if( rc==SQLITE_OK ){
p->bSorted = 1;
}
}else if( p->iId>0 ){
sqlite3_stmt *pDelete = 0;
rc = vec1GetSql(p->pTab, VEC1_SQL_DELETE_FROM_IDX, &pDelete);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDelete, 1, p->iId);
sqlite3_step(pDelete);
vec1StmtReset(&rc, pDelete);
}
if( p->pTab->nMeta>0 && rc==SQLITE_OK ){
rc = vec1GetSql(p->pTab, VEC1_SQL_DELETE_FROM_META, &pDelete);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDelete, 1, p->iId);
sqlite3_step(pDelete);
vec1StmtReset(&rc, pDelete);
}
}
}
return rc;
}
/*
** Return the offset of the first field of vector ii within a blocked
** storage buffer. Each vector is nVectorSize bytes in size (including
** the norm for binary-quantized vectors). szElem0 is the size of the
** first element in each vector (4 for binary-quant, 1 for product-quant).
*/
#define VEC1_LIST_BLOCKED_OFFSET(ii, nVectorSize) ( \
((ii / VEC1_PQ_BLOCKSIZE) * VEC1_PQ_BLOCKSIZE * nVectorSize) + \
(ii % VEC1_PQ_BLOCKSIZE) \
)
/*
** Return the offset of the norm field for vector ii within a blocked
** buffer containing binary quanitized vectors. Each quantized vector
** is nCodebook bytes in size, not including the 4 byte norm.
*/
#define VEC1_LIST_NORM_OFFSET(ii, nCodebook) ( \
((ii / VEC1_PQ_BLOCKSIZE) * VEC1_PQ_BLOCKSIZE * (nCodebook+sizeof_f32)) + \
(ii % VEC1_PQ_BLOCKSIZE) * sizeof_f32 \
)
#define VEC1_LIST_CODE_OFFSET(ii, nCodebook) ( \
((ii / VEC1_PQ_BLOCKSIZE) * VEC1_PQ_BLOCKSIZE * (nCodebook+sizeof_f32)) + \
(VEC1_PQ_BLOCKSIZE * sizeof_f32) + \
(ii % VEC1_PQ_BLOCKSIZE) \
)
/*
** Shuffle the contents of the list-builder around so that it
** contains 0 tombstones.
*/
static int vec1ListBuilderCompress(
Vec1ListBuilder *p,
i64 iDel
){
const int nMeta = p->pTab->nMeta;
int ii;
int iOut = 0;
int nEntry = p->bufRowid.n / p->szRowid;
int *aMetaIn = 0; /* Meta-data input offsets */
int *aMetaOut = 0; /* Meta-data output offsets */
i64 tomb1 = p->szRowid==8 ? VEC1_TOMBSTONE_64 : VEC1_TOMBSTONE_32;
i64 tomb2 = iDel;
/* Allocate space for aMetaIn[] and aMetaOut[] if requried */
if( nMeta>0 ){
aMetaIn = vec1MallocZero(nMeta * 2 * sizeof(int));
if( aMetaIn==0 ) return SQLITE_NOMEM;
aMetaOut = &aMetaIn[nMeta];
for(ii=0; ii<nMeta; ii++){
aMetaIn[ii] = VEC1_META_SZHDR;
aMetaOut[ii] = VEC1_META_SZHDR;
}
}
p->iFirst = VEC1_LARGEST_INT64;
p->iLast = VEC1_SMALLEST_INT64;
for(ii=0; ii<nEntry; ii++){
int iOff = p->szRowid*ii;
i64 iRowid;
int bTombstone = 0;
int iMeta; /* Iterator used for meta-data columns */
if( p->szRowid==8 ){
iRowid = vec1GetU64(&p->bufRowid.a[iOff]);
}else{
iRowid = vec1GetU32(&p->bufRowid.a[iOff]);
}
if( iRowid==tomb1 || iRowid==tomb2 ) bTombstone = 1;
if( bTombstone==0 ){
p->iFirst = MIN(p->iFirst, iRowid);
p->iLast = MAX(p->iLast, iRowid);
if( ii!=iOut ){
int iOff2 = p->szRowid*iOut;
u8 *a = p->bufData.a;
int iFrom, iTo;
memcpy(&p->bufRowid.a[iOff2], &p->bufRowid.a[iOff], p->szRowid);
if( p->eBlocked==VEC1_BLOCKED_NONE ){
int i1 = p->nVectorSize * iOut;
int i2 = p->nVectorSize * ii;
memcpy(&p->bufData.a[i1], &p->bufData.a[i2], p->nVectorSize);
}else{
int nCodebook; /* Size of quantized vector without norm */
int jj;
if( p->eBlocked==VEC1_BLOCKED_BIN ){
nCodebook = p->nVectorSize - sizeof_f32;
memcpy(&a[ VEC1_LIST_NORM_OFFSET(iOut, nCodebook) ],
&a[ VEC1_LIST_NORM_OFFSET(ii, nCodebook) ],
sizeof_f32
);
iFrom = VEC1_LIST_CODE_OFFSET(ii, nCodebook);
iTo = VEC1_LIST_CODE_OFFSET(iOut, nCodebook);
}else{
assert( p->eBlocked==VEC1_BLOCKED_PQ );
nCodebook = p->nVectorSize;
iFrom = VEC1_LIST_BLOCKED_OFFSET(ii, nCodebook);
iTo = VEC1_LIST_BLOCKED_OFFSET(iOut, nCodebook);
}
for(jj=0; jj<nCodebook; jj++){
u8 v = p->bufData.a[iFrom + jj*VEC1_PQ_BLOCKSIZE];
p->bufData.a[iTo + jj*VEC1_PQ_BLOCKSIZE] = v;
}
}
}
iOut++;
}
for(iMeta=0; iMeta<nMeta; iMeta++){
Vec1MetaBuilder *pMeta = &p->aMeta[iMeta];
u8 *a = pMeta->buf.a;
if( pMeta->format==VEC1_META_GENERIC ){
i64 nIn = vec1MetaValueSize(&a[aMetaIn[iMeta]]);
if( nIn<=0 || nIn>pMeta->buf.n-aMetaIn[iMeta] ){
sqlite3_free(aMetaIn);
return VEC1_CORRUPT;
}
if( bTombstone==0 ){
if( aMetaIn[iMeta]!=aMetaOut[iMeta] ){
memmove(&a[ aMetaOut[iMeta] ], &a[ aMetaIn[iMeta] ], nIn);
}
aMetaOut[iMeta] += nIn;
}
aMetaIn[iMeta] += nIn;
}else{
int szElem =
(pMeta->format==VEC1_META_1BYTEINT) ? 1 :
(pMeta->format==VEC1_META_4BYTEINT) ? sizeof_u32 : sizeof_f64
;
if( bTombstone==0 ){
memmove(&a[ aMetaOut[iMeta] ], &a[ aMetaIn[iMeta] ], szElem);
aMetaOut[iMeta] += szElem;
}
aMetaIn[iMeta] += szElem;
}
}
}
p->nTombstone = 0;
p->bufRowid.n = iOut*p->szRowid;
if( p->eBlocked==VEC1_BLOCKED_NONE ){
p->bufData.n = p->nVectorSize * iOut;
}else{
p->bufData.n = p->nVectorSize * VEC1_PQ_BLOCKSIZE * (
((iOut+VEC1_PQ_BLOCKSIZE-1)/VEC1_PQ_BLOCKSIZE)
);
}
for(ii=0; ii<nMeta; ii++){
p->aMeta[ii].buf.n = aMetaOut[ii];
}
sqlite3_free(aMetaIn);
return SQLITE_OK;
}
/*
** Append a rowid/[PQ|vector] pair to a list builder object.
*/
static int vec1ListBuilderAdd(
Vec1ListBuilder *p,
i64 iRowid, /* Rowid value to add to builder */
const u8 *aPQ /* Pointer to buffer containing PQ enc. */
){
int rc = SQLITE_OK;
int nFinal = 0;
const int vs = p->nVectorSize;
assert( p->szRowid==4 || p->szRowid==8 );
/* Check if adding this entry would cause the final blob to be larger
** than the configured 'blocksize'. If so, flush it to disk.
**
** If the proposed rowid may not be added to the current list because
** it is too large or a tombstone value, flush the current list to disk
** in that case as well.
*/
nFinal = p->bufRowid.n + p->bufData.n + p->szRowid + p->nVectorSize;
if( nFinal>p->pTab->cfg.nBlocksize
|| (p->szRowid==4 && (iRowid<0 || iRowid>=VEC1_TOMBSTONE_32))
|| (p->szRowid==8 && iRowid==VEC1_TOMBSTONE_64)
){
rc = vec1ListBuilderFlush(p);
if( rc!=SQLITE_OK ) return rc;
p->szRowid = ((iRowid<0 || iRowid>=VEC1_TOMBSTONE_32) ? 8 : 4);
p->bSorted = 1;
p->iId = 0;
p->nTombstone = 0;
vec1ListBuilderInitMetaArray(p);
}
/* Update Vec1ListBuilder.iFirst, iLast and bSorted to account for the
** rowid value being added by this call. */
if( p->bufRowid.n==0 ){
p->iLast = p->iFirst = iRowid;
}else if( iRowid<p->iLast ){
p->bSorted = 0;
p->iFirst = MIN(p->iFirst, iRowid);
}else{
p->iLast = iRowid;
}
/* Ensure that there is room in the two buffers for this entry. The
** rowids buffer always grows by szRowid bytes. How much the data
** buffer grows by depends on whether it is blocked or not. */
rc = vec1BufferGrow(&p->bufRowid, p->szRowid);
if( rc==SQLITE_OK ){
if( p->eBlocked==VEC1_BLOCKED_NONE ){
rc = vec1BufferGrow(&p->bufData, p->nVectorSize);
}else{
int nEntry = (p->bufRowid.n / p->szRowid);
if( (nEntry % VEC1_PQ_BLOCKSIZE)==0 ){
int nReq = (VEC1_PQ_BLOCKSIZE * p->nVectorSize);
rc = vec1BufferGrow(&p->bufData, nReq);
if( rc==SQLITE_OK ){
memset(&p->bufData.a[p->bufData.n], 0, nReq);
p->bufData.n += nReq;
}
}
}
}
if( rc==SQLITE_OK ){
/* Write the vector or PQ data */
if( p->eBlocked==VEC1_BLOCKED_NONE ){
memcpy(&p->bufData.a[p->bufData.n], aPQ, p->nVectorSize);
p->bufData.n += p->nVectorSize;
}else{
const int iEntry = (p->bufRowid.n / p->szRowid);
const int iBlkOff = (iEntry / VEC1_PQ_BLOCKSIZE) * vs * VEC1_PQ_BLOCKSIZE;
const int iBlkEntry = iEntry % VEC1_PQ_BLOCKSIZE;
int ii = 0;
int iOff = iBlkOff + iBlkEntry;
if( p->eBlocked==VEC1_BLOCKED_BIN ){
/* In this case, the first 4 bytes of the encoded vector is its norm.
** Each block begins with an array of VEC1_PQ_BLOCKSIZE norm values.
** Copy the norm for this vector in now. */
const int iNorm = iBlkOff + (sizeof_f32 * iBlkEntry);
memcpy(&p->bufData.a[iNorm], aPQ, sizeof_f32);
ii = 4;
iOff += (sizeof_f32 * VEC1_PQ_BLOCKSIZE) - ii*VEC1_PQ_BLOCKSIZE;
}
/* Write each byte of the PQ or binary quant code */
for(/* no-op */; ii<p->nVectorSize; ii++){
p->bufData.a[iOff + ii*VEC1_PQ_BLOCKSIZE] = aPQ[ii];
}
}
/* Write the rowid */
if( p->szRowid==8 ){
vec1PutU64(&p->bufRowid.a[p->bufRowid.n], iRowid);
}else{
vec1PutU32(&p->bufRowid.a[p->bufRowid.n], (u32)iRowid);
}
p->bufRowid.n += p->szRowid;
}
return rc;
}
static void vec1ListBuilderFree(Vec1ListBuilder *p){
int ii;
vec1BufferFree(&p->bufRowid);
vec1BufferFree(&p->bufData);
if( p->aMeta ){
for(ii=0; ii<p->pTab->nMeta; ii++){
vec1BufferFree(&p->aMeta[ii].buf);
}
}
sqlite3_free(p->aMeta);
}
/*
** Allocate a new writer object for vec1 table pTab.
*/
static int vec1WriterAlloc(
Vec1Tab *pTab, /* Vec1 table */
int bRebuild, /* True if this is 'rebuild' op */
Vec1Writer **pp /* OUT: New writer object */
){
int rc = SQLITE_OK;
Vec1Writer *p = 0;
int nBld = pTab->mod.hdr.nBucket ? pTab->mod.hdr.nBucket : 1;
int nEncoded = vec1EncodedVectorSize(&pTab->mod);
int nByte =
sizeof(Vec1Writer) + /* Vec1Writer */
sizeof(Vec1ListBuilder) * nBld + /* Vec1Writer.aBld */
sizeof_f32 * pTab->nTmpVec + /* Vec1Writer.aResidual */
nEncoded; /* Vec1Writer.aPQ */
p = (Vec1Writer*)vec1MallocZero(nByte);
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
memset(p, 0, nByte);
p->aBld = (Vec1ListBuilder*)&p[1];
p->aResidual = (float*)&p->aBld[nBld];
p->aPQ = (u8*)&p->aResidual[pTab->nTmpVec];
p->pTab = pTab;
p->bRebuild = bRebuild;
p->nBld = nBld;
if( bRebuild ){
int ii;
for(ii=0; rc==SQLITE_OK && ii<nBld; ii++){
rc = vec1ListBuilderInit(&p->aBld[ii], pTab, ii);
}
}
}
*pp = p;
return rc;
}
/*
** Quantize a vector so that it can be written to the index.
*/
static void vec1QuantizeVector(
const Vec1Model *pMod, /* Current model */
float *aTmp, /* Temporary space - same dim as vectors */
const float *aVector, /* Vector to quantize */
int *piBucket, /* OUT: Bucket to put vector in */
u8 *aCode /* OUT: Write PQ code (if any) here */
){
const int nBucket = pMod->hdr.nBucket;
const float *aVec;
int iBucket = 0;
aVec = vec1TransformInputVector(pMod, aTmp, aVector);
if( nBucket>0 ){
int nElem = pMod->hdr.nElem;
iBucket = vec1BestMatchSimple(pMod->aCentroid, nBucket, aVec, nElem, 0);
if( pMod->hdr.nCodebook>0 && (pMod->hdr.flags & VEC1_MODEL_RESIDUAL) ){
vec1Sub(aTmp, aVec, &pMod->aCentroid[iBucket*nElem], nElem);
aVec = aTmp;
}
}
if( pMod->hdr.nCodebook>0 ){
vec1EncodeVector(pMod, aVec, aCode);
}
*piBucket = iBucket;
}
static int vec1WriterVector(
Vec1Writer *p, /* Writer object to write to */
i64 iRowid, /* Rowid of new entry */
const float *aVector, /* New vector */
int *piBucket
){
Vec1Tab *pTab = p->pTab;
const u8 *aStore;
Vec1ListBuilder *pBld = 0;
const Vec1Model *pMod = &pTab->mod;
int rc = SQLITE_OK;
int iBld = *piBucket;
/* Transform and quantize the vector */
if( iBld<0 ){
vec1QuantizeVector(pMod, p->aResidual, aVector, &iBld, p->aPQ);
}
if( pMod->hdr.nCodebook>0 ){
aStore = p->aPQ;
}else{
/* If the index is storing full vectors, not PQ codes, store the original,
** not the transformed vector. */
aStore = (const u8*)aVector;
}
pBld = &p->aBld[iBld];
if( pBld->pTab==0 ){
/* Attempt to load blob smaller than 'blocksize-min' belonging to
** this bucket to append to. */
sqlite3_stmt *pSearch = 0;
rc = vec1ListBuilderInit(pBld, pTab, iBld);
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_SEARCH_IDX_FOR_INSERT, &pSearch);
}
if( rc==SQLITE_OK ){
sqlite3_bind_int(pSearch, 1, iBld);
sqlite3_bind_int(pSearch, 2, pTab->cfg.nBlocksizeMin);
if( SQLITE_ROW==sqlite3_step(pSearch) ){
i64 iId = sqlite3_column_int64(pSearch, 0);
i64 iFirst = sqlite3_column_int64(pSearch, 1);
i64 iLast = sqlite3_column_int64(pSearch, 2);
const u8 *aVal = (const u8*)sqlite3_column_blob(pSearch, 3);
int nVal = sqlite3_column_bytes(pSearch, 3);
rc = vec1ListBuilderLoad(pBld, iId, iFirst, iLast, aVal, nVal);
}
vec1StmtReset(&rc, pSearch);
}
}
if( rc==SQLITE_OK ){
rc = vec1ListBuilderAdd(pBld, iRowid, aStore);
}
*piBucket = iBld;
return rc;
}
/*
** Serialize value pVal and append it to buffer pBuf. Vec1 uses a serialization
** format based on SQLite's. First a varint indicating the type and size
** of the value, followed by the data for the value itself. Text values
** are always encoded using utf-8.
**
** 0: NULL value. 0 byte payload.
** 1: Integer value (unsigned). 1 byte payload.
** 2: Integer value (signed). 4 byte payload.
** 3: Integer value (signed). 8 byte payload.
** 4: Real value. 8 byte payload.
**
** Greater than 4 and odd is a text value. Payload size ((v-5)/2). Greater
** than 4 and even is a blob value. Payload size ((v-6)/2).
*/
static int vec1AppendMetaValue(
Vec1Buffer *pBuf,
sqlite3_value *pVal
){
int rc = SQLITE_OK;
switch( sqlite3_value_type(pVal) ){
case SQLITE_NULL: {
rc = vec1BufferGrow(pBuf, 1);
if( rc==SQLITE_OK ){
pBuf->a[pBuf->n++] = 0x00;
}
break;
}
case SQLITE_INTEGER: {
i64 iVal = sqlite3_value_int64(pVal);
rc = vec1BufferGrow(pBuf, 9);
if( rc==SQLITE_OK ){
if( iVal>=0 && iVal<=254 ){
pBuf->a[pBuf->n++] = 0x01;
pBuf->a[pBuf->n++] = (iVal & 0xFF);
}
else if( iVal>=VEC1_META_4BYTEMIN && iVal<=VEC1_META_4BYTEMAX ){
pBuf->a[pBuf->n++] = 0x02;
vec1PutU32(&pBuf->a[pBuf->n], (int)iVal);
pBuf->n += 4;
}else{
pBuf->a[pBuf->n++] = 0x03;
vec1PutU64(&pBuf->a[pBuf->n], iVal);
pBuf->n += 8;
}
}
break;
}
case SQLITE_FLOAT: {
double fVal = sqlite3_value_double(pVal);
rc = vec1BufferGrow(pBuf, 9);
if( rc==SQLITE_OK ){
i64 iVal;
memcpy(&iVal, &fVal, sizeof(fVal));
pBuf->a[pBuf->n++] = 0x04;
vec1PutU64(&pBuf->a[pBuf->n], iVal);
pBuf->n += 8;
}
break;
}
case SQLITE_TEXT: {
const char *z = (const char*)sqlite3_value_text(pVal);
int n = sqlite3_value_bytes(pVal);
rc = vec1BufferGrow(pBuf, n+5);
if( rc==SQLITE_OK ){
pBuf->n += vec1PutVarint(&pBuf->a[pBuf->n], (n*2 + 5));
memcpy(&pBuf->a[pBuf->n], z, n);
pBuf->n += n;
}
break;
}
default: {
const u8 *z = (const u8*)sqlite3_value_blob(pVal);
int n = sqlite3_value_bytes(pVal);
rc = vec1BufferGrow(pBuf, n+5);
if( rc==SQLITE_OK ){
pBuf->n += vec1PutVarint(&pBuf->a[pBuf->n], (n*2 + 6));
memcpy(&pBuf->a[pBuf->n], z, n);
pBuf->n += n;
}
assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
break;
}
}
return rc;
}
static void vec1MetaValueUpdateFlags(const u8 *aMeta, u32 *pFlags){
switch( aMeta[0] ){
case 0: /* NULL value */
*pFlags |= VEC1_META_HASNULL;
break;
case 1: /* 1-byte integer */
break;
case 2: /* 4-byte integer */
*pFlags &= ~(VEC1_META_1BYTEINT);
break;
case 4: /* 8-byte real */
*pFlags &= ~(VEC1_META_1BYTEINT|VEC1_META_4BYTEINT);
break;
default:
*pFlags &= ~(VEC1_META_1BYTEINT|VEC1_META_4BYTEINT|VEC1_META_REAL);
break;
}
}
/*
** The meta-list builder passed as the only argument currently contains
** a list of values in non-generic format (e.g. VEC1_META_1BYTEINT format).
** This function converts the list to generic format.
**
** Return SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM)
** otherwise.
*/
static int vec1ExpandMetaValue(Vec1MetaBuilder *p){
Vec1Buffer orig;
int rc = SQLITE_OK;
int bHasNull = 0;
memcpy(&orig, &p->buf, sizeof(Vec1Buffer));
p->buf.a = 0;
p->buf.n = VEC1_META_SZHDR;
p->buf.nAlloc = 0;
assert( p->format==VEC1_META_1BYTEINT
|| p->format==VEC1_META_4BYTEINT
|| p->format==VEC1_META_REAL
);
rc = vec1BufferGrow(&p->buf, (orig.n * 2));
if( rc==SQLITE_OK ){
int iIn = VEC1_META_SZHDR;
int iOut = VEC1_META_SZHDR;
while( iIn<orig.n ){
if( p->format==VEC1_META_1BYTEINT ){
int v = orig.a[iIn++];
if( v==VEC1_META_1BYTENULL ){
p->buf.a[iOut++] = 0x00;
bHasNull = 1;
}else{
p->buf.a[iOut++] = 0x01;
p->buf.a[iOut++] = (v & 0xFF);
}
}else if( p->format==VEC1_META_4BYTEINT ){
u32 v = vec1GetU32(&orig.a[iIn]);
iIn += sizeof_u32;
if( v==VEC1_META_4BYTENULL ){
p->buf.a[iOut++] = 0x00;
bHasNull = 1;
}else{
p->buf.a[iOut++] = 0x02;
vec1PutU32(&p->buf.a[iOut], (u32)v);
iOut += sizeof_u32;
}
}else{
u64 v = vec1GetU64(&orig.a[iIn]);
iIn += sizeof_f64;
if( v==VEC1_META_REALNULL ){
p->buf.a[iOut++] = 0x00;
bHasNull = 1;
}else{
p->buf.a[iOut++] = 0x04;
vec1PutU64(&p->buf.a[iOut], v);
iOut += sizeof_f64;
}
}
}
p->buf.n = iOut;
}
if( p->format==VEC1_META_1BYTEINT ){
p->flags = VEC1_META_4BYTEINT | VEC1_META_1BYTEINT | VEC1_META_REAL;
}else if( p->format==VEC1_META_4BYTEINT ){
p->flags = VEC1_META_4BYTEINT | VEC1_META_REAL;
}else{
p->flags = VEC1_META_REAL;
}
p->format = VEC1_META_GENERIC;
if( bHasNull ){
p->flags |= VEC1_META_HASNULL;
}
vec1BufferFree(&orig);
return rc;
}
static int vec1WriterMetaFromArray(
Vec1Writer *pWriter, /* Writer object accumulating changes */
int iBucket, /* Bucket to write to */
sqlite3_value **apMeta /* Meta values associated with prev vector */
){
Vec1ListBuilder *p = &pWriter->aBld[iBucket];
int ii;
int rc = SQLITE_OK;
assert( p->pTab!=0 );
for(ii=0; rc==SQLITE_OK && ii<p->pTab->nMeta; ii++){
Vec1MetaBuilder *pMeta = &p->aMeta[ii];
Vec1Buffer *pBuf = &pMeta->buf;
sqlite3_value *pVal = apMeta[ii];
if( pMeta->format==VEC1_META_1BYTEINT
|| pMeta->format==VEC1_META_4BYTEINT
|| pMeta->format==VEC1_META_REAL
){
int eType = sqlite3_value_type(pVal);
rc = vec1BufferGrow(pBuf, sizeof_f64);
if( rc!=SQLITE_OK ) continue;
if( eType==SQLITE_INTEGER ){
i64 iVal = sqlite3_value_int64(pVal);
if( pMeta->format==VEC1_META_1BYTEINT ){
if( iVal>=0 && iVal<=254 ){
pBuf->a[pBuf->n++] = (iVal & 0xFF);
pVal = 0;
}
}else{
if( iVal>=VEC1_META_4BYTEMIN && iVal<=VEC1_META_4BYTEMAX ){
if( pMeta->format==VEC1_META_4BYTEINT ){
vec1PutU32(&pBuf->a[pBuf->n], (u32)iVal);
pBuf->n += sizeof_u32;
}else{
double fVal = (double)iVal;
u64 v;
memcpy(&v, &fVal, sizeof_f64);
vec1PutU64(&pBuf->a[pBuf->n], v);
pBuf->n += sizeof_f64;
}
pVal = 0;
}
}
}else if( eType==SQLITE_FLOAT && pMeta->format==VEC1_META_REAL ){
double fVal = sqlite3_value_double(pVal);
u64 v;
memcpy(&v, &fVal, sizeof_f64);
vec1PutU64(&pBuf->a[pBuf->n], v);
pBuf->n += sizeof_f64;
pVal = 0;
}else if( eType==SQLITE_NULL ){
rc = vec1BufferGrow(pBuf, sizeof_f64);
if( pMeta->format==VEC1_META_1BYTEINT ){
pBuf->a[pBuf->n++] = (VEC1_META_1BYTENULL & 0xFF);
}
else if( pMeta->format==VEC1_META_4BYTEINT ){
vec1PutU32(&pBuf->a[pBuf->n], (u32)VEC1_META_4BYTENULL);
pBuf->n += sizeof_u32;
}else{
vec1PutU64(&pBuf->a[pBuf->n], (u64)VEC1_META_REALNULL);
pBuf->n += sizeof_f64;
}
pVal = 0;
}
if( pVal ){
rc = vec1ExpandMetaValue(pMeta);
}
}
if( pMeta->format==VEC1_META_GENERIC ){
int iOff = pBuf->n;
rc = vec1AppendMetaValue(pBuf, pVal);
if( rc==SQLITE_OK ){
vec1MetaValueUpdateFlags(&pBuf->a[iOff], &pMeta->flags);
}
}
}
return rc;
}
static int vec1WriterMetaFromStmt(
Vec1Writer *pWriter, /* Writer object accumulating changes */
int iBucket, /* Bucket to write to */
sqlite3_stmt *pStmt
){
Vec1ListBuilder *p = &pWriter->aBld[iBucket];
int ii;
int rc = SQLITE_OK;
assert( p->pTab!=0 );
for(ii=0; rc==SQLITE_OK && ii<p->pTab->nMeta; ii++){
Vec1Buffer *pBuf = &p->aMeta[ii].buf;
sqlite3_value *pVal = sqlite3_column_value(pStmt, ii+2);
int iOff = pBuf->n;
rc = vec1AppendMetaValue(pBuf, pVal);
if( rc==SQLITE_OK ){
vec1MetaValueUpdateFlags(&pBuf->a[iOff], &p->aMeta[ii].flags);
}
}
return rc;
}
/*
** Read nMeta values from buffer pBuf. Use them as the meta values associated
** with the (compressed) vector just appended to bucket iBucket.
**
** This is only called on a buffer that has been constructed in process,
** not one loaded from disk, so it is safe to assume that pBuf really does
** contain nMeta well-formed values.
*/
static int vec1WriterMetaFromPacked(
Vec1Writer *pWriter, /* Writer object accumulating changes */
int iBucket, /* Bucket to write to */
Vec1Buffer *pBuf,
int *piOff /* IN/OUT: Buffer offset */
){
int ii;
int iOff = *piOff;
for(ii=0; ii<pWriter->pTab->nMeta; ii++){
Vec1MetaBuilder *pTo = &pWriter->aBld[iBucket].aMeta[ii];
i64 n = vec1MetaValueSize(&pBuf->a[iOff]);
int rc = vec1BufferGrow(&pTo->buf, n);
vec1MetaValueUpdateFlags(&pBuf->a[iOff], &pTo->flags);
if( rc!=SQLITE_OK ) return rc;
memcpy(&pTo->buf.a[pTo->buf.n], &pBuf->a[iOff], n);
pTo->buf.n += n;
iOff += n;
}
*piOff = iOff;
return SQLITE_OK;
}
static int vec1PackMetaValues(
Vec1Tab *pTab,
Vec1Buffer *pBuf,
sqlite3_stmt *pStmt
){
int ii;
int rc = SQLITE_OK;
for(ii=0; rc==SQLITE_OK && ii<pTab->nMeta; ii++){
rc = vec1AppendMetaValue(pBuf, sqlite3_column_value(pStmt, 2+ii));
}
return rc;
}
static int vec1WriterQuantized(
Vec1Writer *pWriter,
i64 iRowid,
int iBucket,
const u8 *aStore
){
return vec1ListBuilderAdd(&pWriter->aBld[iBucket], iRowid, aStore);
}
static int vec1WriterFinish(Vec1Writer *p, int rcin){
int rc = rcin;
if( p ){
int ii;
for(ii=0; ii<p->nBld; ii++){
if( rc==SQLITE_OK ){
rc = vec1ListBuilderFlush(&p->aBld[ii]);
}
vec1ListBuilderFree(&p->aBld[ii]);
}
sqlite3_free(p);
}
return rc;
}
static void vec1EncodeVectorN(
const Vec1Model *pMod,
const float *aVec[VEC1_MULTIMATCH_NVEC],
u8 *aCode
){
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ){
int nCode = vec1EncodedVectorSize(pMod);
int vv;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
vec1EncodeVector(pMod, aVec[vv], &aCode[vv*nCode]);
}
}else{
const int nCodebook = pMod->hdr.nCodebook;
const int nCodeElem = pMod->nCodeElem;
const float *aSub[VEC1_MULTIMATCH_NVEC];
int vv;
int k;
for(k=0; k<nCodebook; k++){
int aBest[VEC1_MULTIMATCH_NVEC];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aSub[vv] = &aVec[vv][k*nCodeElem];
}
vec1BestMatchPlatformN(
nCodeElem,
&pMod->aModel[k * nCodeElem * VEC1_PQ_CODEBOOK_SZ],
&pMod->aModelNT[k * nCodeElem * VEC1_PQ_CODEBOOK_SZ],
VEC1_PQ_CODEBOOK_SZ, aSub, aBest, 0
);
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aCode[vv*nCodebook + k] = (u8)aBest[vv];
}
}
}
}
static void vec1TransformInputVectorN(
const Vec1Model *pMod, /* Model to transform according to */
const float *aIn,
const float *aOut[VEC1_MULTIMATCH_NVEC],
float *aTmp, /* Temp space for vectors, if required */
int nStride /* Pad output vectors to this many fields */
){
const int nElem = pMod->hdr.nElem;
int vv;
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aOut[vv] = &aIn[vv*nElem];
}
if( (pMod->hdr.flags & VEC1_MODEL_ROTATE) ){
float *aRot[VEC1_MULTIMATCH_NVEC];
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
aRot[vv] = &aTmp[vv*nStride];
}
#if defined(VEC1_USE_TRANSPOSITION)
vec1RotateVectorTransposeN(
aRot, aOut, nElem, pMod->aRotation, pMod->aRotationNT
);
#else
vec1RotateVectorN(aRot, aOut, nElem, pMod->aRotation);
#endif
memcpy(aOut, aRot, sizeof(aRot));
}
else if( (pMod->hdr.flags & VEC1_MODEL_WHT) ){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float *tmp = &aTmp[vv*nStride];
memcpy(tmp, aOut[vv], nElem * sizeof_f32);
vec1ApplyWHT(tmp, nElem);
aOut[vv] = tmp;
}
}
else if( pMod->hdr.eDistance==VEC1_DISTANCE_COS || nStride!=nElem ){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float *tmp = &aTmp[vv*nStride];
memcpy(tmp, aOut[vv], nElem * sizeof_f32);
aOut[vv] = tmp;
}
}
/* If this model uses cosine distance, normalize the vectors. */
if( pMod->hdr.eDistance==VEC1_DISTANCE_COS ){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
assert( aOut[vv]==&aTmp[vv*nStride] );
vec1NormalizeVector(&aTmp[vv*nStride], nElem);
}
}
}
/*
** Quantize N vectors so that they can be written to the index. N is always
** VEC1_MULTIMATCH_NVEC.
*/
static void vec1QuantizeVectorN(
const Vec1Model *pMod, /* Current model */
float *aTmp, /* Temporary space - same dim as vectors */
const float *aVector, /* Vector to quantize */
int *aBucket, /* OUT: Buckets to put vector in */
u8 *aCode /* OUT: Write PQ codes (if any) here */
){
int nElem = pMod->hdr.nElem;
const int nBucket = pMod->hdr.nBucket;
const int nStride = MAX(nElem, pMod->nCodeElem * (int)pMod->hdr.nCodebook);
const float *aVec[VEC1_MULTIMATCH_NVEC];
int vv;
vec1TransformInputVectorN(pMod, aVector, aVec, aTmp, nStride);
if( nBucket>0 ){
vec1BestMatchPlatformN(
nElem, pMod->aCentroid, pMod->aCentroidNT, nBucket, aVec, aBucket, 0
);
if( pMod->hdr.nCodebook>0 && (pMod->hdr.flags & VEC1_MODEL_RESIDUAL) ){
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
float *tmp = &aTmp[vv*nStride];
vec1Sub(tmp, aVec[vv], &pMod->aCentroid[aBucket[vv]*nElem], nElem);
aVec[vv] = tmp;
}
}
for(vv=0; vv<VEC1_MULTIMATCH_NVEC; vv++){
assert( aBucket[vv]>=0 && aBucket[vv]<nBucket );
}
}else{
memset(aBucket, 0, sizeof(int)*VEC1_MULTIMATCH_NVEC);
}
if( pMod->hdr.nCodebook>0 ){
vec1EncodeVectorN(pMod, aVec, aCode);
}
}
/*
** Each Vec1QuantizeJob has capacity for this many vectors.
*/
#define VEC1_QUANTIZE_JOB_SZ 1024
typedef struct Vec1QuantizeJob Vec1QuantizeJob;
struct Vec1QuantizeJob {
Vec1Model *pModel;
Vec1Writer *pWriter;
int nVector; /* Number of vectors to quantize */
i64 *aRowid; /* Array of nVector rowids */
float *aTmp; /* Temp space for one vector */
float *aVec; /* Packed vectors */
int *aBucket; /* OUT: Bucket for each vector */
u8 *aCode; /* OUT: Packed codes for vectors */
int nEncoded; /* Size of each encoded vector in bytes */
Vec1Buffer meta; /* Packed array of meta values */
};
static void vec1QuantizeJob(void *pCtx){
Vec1QuantizeJob *p = (Vec1QuantizeJob*)pCtx;
const int nQuant = VEC1_MULTIMATCH_NVEC;
const int nElem = p->pModel->hdr.nElem;
int ii = 0;
/* Handle the bulk of the vectors VEC1_QUANTIZE_JOB_QUANTA at a time. */
for(; ii<=(p->nVector-nQuant); ii += nQuant){
float *v = &p->aVec[nElem * ii];
u8 *a = &p->aCode[p->nEncoded * ii];
vec1QuantizeVectorN(p->pModel, p->aTmp, v, &p->aBucket[ii], a);
}
/* Scalar tail - handle remaining vectors one at a time. */
for(; ii<p->nVector; ii++){
float *v = &p->aVec[nElem * ii];
u8 *a = &p->aCode[p->nEncoded * ii];
vec1QuantizeVector(p->pModel, p->aTmp, v, &p->aBucket[ii], a);
}
}
static int vec1UpdateBase(
Vec1Tab *pTab,
i64 iId,
int iBucket
){
sqlite3_stmt *pStmt = 0;
int rc = vec1GetSql(pTab, VEC1_SQL_UPDATE_BASE2, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, iBucket);
sqlite3_bind_int64(pStmt, 2, iId);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
}
return rc;
}
static int vec1QuantizeJobFinish(void *pCtx, int rcin){
Vec1QuantizeJob *p = (Vec1QuantizeJob*)pCtx;
const int nElem = p->pModel->hdr.nElem;
int rc = rcin;
int ii;
int bBucketBase = vec1ModelIsFlat(p->pModel);
int iOffMeta = 0;
assert( bBucketBase==0 || p->pModel->hdr.nCodebook==0 );
for(ii=0; rc==SQLITE_OK && ii<p->nVector; ii++){
int iBucket = p->aBucket[ii];
const u8 *aStore;
if( p->nEncoded>0 ){
aStore = &p->aCode[ii * p->nEncoded];
}else{
aStore = (const u8*)&p->aVec[ii * nElem];
}
rc = vec1WriterQuantized(p->pWriter, p->aRowid[ii], iBucket, aStore);
if( bBucketBase && rc==SQLITE_OK ){
rc = vec1UpdateBase(p->pWriter->pTab, p->aRowid[ii], iBucket);
}
if( rc==SQLITE_OK ){
rc = vec1WriterMetaFromPacked(p->pWriter, iBucket, &p->meta, &iOffMeta);
}
}
vec1BufferFree(&p->meta);
sqlite3_free(p);
return rc;
}
static int vec1UpdateBaseRows(
Vec1Tab *pTab,
i64 iFirst,
i64 iLast
){
sqlite3_stmt *pStmt = 0;
int rc = vec1GetSql(pTab, VEC1_SQL_ZERO_BASE_RANGE, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pStmt, 1, iFirst);
sqlite3_bind_int64(pStmt, 2, iLast);
sqlite3_step(pStmt);
vec1StmtReset(&rc, pStmt);
}
return rc;
}
#define VEC1_ROWS_PER_BASE_UPDATE 64
/*
** Rebuild the entire ANN index from scratch, based on the current contents
** of the %_base table.
*/
static int vec1RebuildIndex(Vec1Tab *pTab, int nThread){
const int nElem = pTab->cfg.nElem;
sqlite3_stmt *pStmt = 0;
int rc = SQLITE_OK;
int bZeroBase = 0; /* True to set base.vector=0 */
Vec1Writer *p = 0; /* Write buffer object */
Vec1JobQueue *pQueue = 0; /* Job queue for multi-threaded rebuilds */
Vec1QuantizeJob *pQJ = 0;
int nRow = 0;
i64 iFirstInRange = VEC1_SMALLEST_INT64;
assert( pTab->cfg.nElem==(int)pTab->mod.hdr.nElem || pTab->mod.hdr.nElem==0 );
/* Clear any existing ANN index */
rc = vec1SqlExec(pTab, "DELETE FROM %Q.'%q_idx'", 0);
if( rc==SQLITE_OK ){
rc = vec1SqlExec(pTab, "DELETE FROM %Q.'%q_meta'", 0);
}
if( rc!=SQLITE_OK || (pTab->mod.hdr.flags & VEC1_MODEL_INDEX)==0 ) return rc;
/* Allocate a writer object. Create transposed centroids if required. */
rc = vec1WriterAlloc(pTab, 1, &p);
if( rc==SQLITE_OK ){
rc = vec1ModelAllocTCent(&pTab->mod);
}
/* If multiple threads have been configured and the model is not 'flat',
** create a job queue so that the rebuild can use multiple threads. There
** is nothing for background threads to do for 'flat' indexes. */
if( vec1ModelIsFlat(&pTab->mod) && pTab->mod.hdr.nBucket==0 ){
bZeroBase = 1;
}
#if VEC1_THREADS
if( rc==SQLITE_OK && nThread>1 && bZeroBase==0 ){
pQueue = vec1JobQueueNew(nThread-1);
if( pQueue==0 ){
rc = SQLITE_NOMEM;
}
}
#endif /* VEC1_THREADS */
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_SCAN_BASE, &pStmt);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
i64 id = sqlite3_column_int64(pStmt, 0);
int nVec = 0;
const float *aVec = 0;
nRow++;
if( bZeroBase && nRow==VEC1_ROWS_PER_BASE_UPDATE ){
rc = vec1UpdateBaseRows(pTab, iFirstInRange, id);
if( rc!=SQLITE_OK ) break;
iFirstInRange = id+1;
nRow = 0;
}
nVec = sqlite3_column_bytes(pStmt, 1);
aVec = (const float*)sqlite3_column_blob(pStmt, 1);
if( nVec!=(sizeof_f32*nElem) ){
rc = SQLITE_ERROR;
vec1VtabError(
pTab, "vec1: unexpected vector size in %q_base: %d", pTab->zName, nVec
);
}else{
if( bZeroBase ){
int iBucket = -1; /* Must be -1 for vec1WriterVector()! */
rc = vec1WriterVector(p, id, aVec, &iBucket);
if( rc==SQLITE_OK ){
rc = vec1WriterMetaFromStmt(p, iBucket, pStmt);
}
}else{
if( pQJ==0 ){
int nEncoded = vec1EncodedVectorSize(&pTab->mod);
int nByte = sizeof(Vec1QuantizeJob) +
sizeof(i64) * VEC1_QUANTIZE_JOB_SZ + /* aRowid */
pTab->nTmpVec * sizeof_f32 * VEC1_MULTIMATCH_NVEC +/* aTmp */
nElem * sizeof_f32 * VEC1_QUANTIZE_JOB_SZ + /* aVec */
sizeof(int) * VEC1_QUANTIZE_JOB_SZ + /* aBucket */
nEncoded * VEC1_QUANTIZE_JOB_SZ; /* aCode */
pQJ = (Vec1QuantizeJob*)sqlite3_malloc(nByte);
if( pQJ==0 ){
rc = SQLITE_NOMEM;
break;
}else{
memset(pQJ, 0, sizeof(Vec1QuantizeJob));
pQJ->pModel = &pTab->mod;
pQJ->pWriter = p;
pQJ->aRowid = (i64*)&pQJ[1];
pQJ->aTmp = (float*)&pQJ->aRowid[VEC1_QUANTIZE_JOB_SZ];
pQJ->aVec = &pQJ->aTmp[pTab->nTmpVec * VEC1_MULTIMATCH_NVEC];
pQJ->aBucket = (int*)&pQJ->aVec[nElem * VEC1_QUANTIZE_JOB_SZ];
pQJ->aCode = (u8*)&pQJ->aBucket[VEC1_QUANTIZE_JOB_SZ];
pQJ->nEncoded = nEncoded;
memset(pQJ->aTmp, 0, sizeof_f32*pTab->nTmpVec*VEC1_MULTIMATCH_NVEC);
}
}
memcpy(&pQJ->aVec[nElem * pQJ->nVector], aVec, nVec);
pQJ->aRowid[pQJ->nVector] = id;
pQJ->nVector++;
rc = vec1PackMetaValues(pTab, &pQJ->meta, pStmt);
if( rc==SQLITE_OK && pQJ->nVector==VEC1_QUANTIZE_JOB_SZ ){
rc = vec1JobQueueAddJob(
pQueue, vec1QuantizeJob, vec1QuantizeJobFinish, (void*)pQJ
);
pQJ = 0;
}
}
}
}
if( rc==SQLITE_OK && bZeroBase ){
rc = vec1UpdateBaseRows(pTab, iFirstInRange, VEC1_LARGEST_INT64);
}
if( pQJ ){
if( rc==SQLITE_OK ){
rc = vec1JobQueueAddJob(
pQueue, vec1QuantizeJob, vec1QuantizeJobFinish, (void*)pQJ
);
}else{
vec1BufferFree(&pQJ->meta);
sqlite3_free(pQJ);
}
}
rc = vec1JobQueueFinishJobs(pQueue, rc);
vec1JobQueueFree(pQueue);
rc = vec1WriterFinish(p, rc);
vec1StmtReset(&rc, pStmt);
vec1ModelFreeTCent(&pTab->mod);
return rc;
}
/*
** The table currently has a flat index installed - an index with
** nCodesize==0 - and bucket numbers instead of vectors in the %_base
** table. This function scans the index, copying full vectors back into
** the %_base table and deleting %_idx entries as it goes.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise.
*/
static int vec1FlatToNone(Vec1Tab *pTab){
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
sqlite3_stmt *pUp = 0;
sqlite3_stmt *pDel = 0;
rc = vec1GetSql(pTab, VEC1_SQL_SCAN_IDX, &pStmt);
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_UPDATE_BASE, &pUp);
}
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_DELETE_FROM_IDX, &pDel);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
Vec1FlatIter iter;
const u8 *aBlob = (const u8*)sqlite3_column_blob(pStmt, 0);
int nBlob = sqlite3_column_bytes(pStmt, 0);
i64 iRowid = sqlite3_column_int64(pStmt, 2);
rc = vec1FlatIterStart(pTab, &iter, aBlob, nBlob);
while( rc==SQLITE_OK && iter.aVec!=0 ){
rc = sqlite3_bind_blob(pUp, 1, iter.aVec, iter.szVec, SQLITE_STATIC);
sqlite3_bind_int64(pUp, 2, iter.iRowid);
sqlite3_step(pUp);
vec1StmtReset(&rc, pUp);
vec1FlatIterNext(&iter);
}
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pDel, 1, iRowid);
sqlite3_step(pDel);
vec1StmtReset(&rc, pDel);
}
}
if( pUp ) sqlite3_clear_bindings(pUp);
vec1StmtReset(&rc, pStmt);
return rc;
}
/*
** Symbols for the commands the vtab accepts. These must match the array
** in vec1InterpretCmdName().
*/
#define VEC1_CMD_BLOCKSIZE 0
#define VEC1_CMD_REBUILD 1
#define VEC1_CMD_BLOCKSIZE_MIN 2
static int vec1InterpretCmdName(Vec1Tab *pTab, const char *zCmd, int *piCmd){
const char *azCmd[] = {
"blocksize",
"rebuild",
"blocksize-min",
0
};
int iCmd;
for(iCmd=0; iCmd<size_of_array(azCmd); iCmd++){
if( 0==sqlite3_stricmp(zCmd, azCmd[iCmd]) ){
*piCmd = iCmd;
return SQLITE_OK;
}
}
vec1VtabError(pTab, "vec1: unknown command: %s", zCmd);
return SQLITE_ERROR;
}
static int vec1InterpretInteger(
Vec1Tab *pTab,
sqlite3_value *pVal,
const char *zName,
i64 iMin,
i64 iMax
){
if( sqlite3_value_numeric_type(pVal)==SQLITE_INTEGER ){
i64 iVal = sqlite3_value_int64(pVal);
if( iVal>=iMin && iVal<=iMax ) return SQLITE_OK;
}
vec1VtabError(
pTab, "'%s' requires an integer value between %lld and %lld",
zName, iMin, iMax
);
return SQLITE_ERROR;
}
static int vec1WriteConfigI64(Vec1Tab *pTab, int iField, i64 iVal){
sqlite3_stmt *pStmt = 0;
int rc = SQLITE_OK;
rc = vec1GetSql(pTab, VEC1_SQL_WRITE_CONFIG, &pStmt);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, iField);
sqlite3_bind_int64(pStmt, 2, iVal);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
}
return rc;
}
/*
** Fix the vector size to nVec *bytes*. This is done when the first
** vector is inserted into the table. Return SQLITE_OK if successful, or
** an SQLite error code if an error occurs.
*/
static int vec1FixVectorSize(Vec1Tab *pTab, int nVec){
int rc = SQLITE_OK;
if( (nVec % sizeof_f32)
|| (nVec<(VEC1_VECSIZE_MIN*sizeof_f32))
|| (nVec>(VEC1_VECSIZE_MAX*sizeof_f32))
){
vec1VtabError(pTab, "invalid vector size: %d bytes", nVec);
rc = SQLITE_ERROR;
}else{
int nElem = nVec/sizeof_f32;
rc = vec1WriteConfigI64(pTab, VEC1_CONFIG_NELEM, nElem);
if( rc==SQLITE_OK ) pTab->cfg.nElem = nElem;
}
return rc;
}
/*
** Handle a "special" INSERT command on a vec1 table. A special INSERT takes
** the form:
**
** INSERT INTO vec1_table(cmd, vector) VALUES('cmd-name', pArg);
**
** The zCmd argument to this function is the text value specified for column
** "cmd" in the INSERT statement. Parameter pArg is the pArg value from the
** INSERT.
**
** Return SQLITE_OK if successful, or an SQLite error code otherwise. If
** an error code is returned, an error message may be left in pTab.
*/
static int vec1SpecialInsert(
Vec1Tab *pTab,
const char *zCmd,
sqlite3_value *pArg
){
int rc = SQLITE_OK;
int iCmd = 0;
rc = vec1InterpretCmdName(pTab, zCmd, &iCmd);
if( rc==SQLITE_OK ){
switch( iCmd ){
case VEC1_CMD_BLOCKSIZE: {
rc = vec1InterpretInteger(
pTab, pArg, "blocksize", VEC1_BLOCKSIZE_MIN, VEC1_BLOCKSIZE_MAX
);
if( rc==SQLITE_OK ){
rc = vec1WriteConfigI64(
pTab, VEC1_CONFIG_BLOCKSIZE, sqlite3_value_int64(pArg)
);
}
break;
};
case VEC1_CMD_BLOCKSIZE_MIN: {
rc = vec1InterpretInteger(
pTab, pArg, "blocksize-min", VEC1_BLOCKSIZE_MIN, VEC1_BLOCKSIZE_MAX
);
if( rc==SQLITE_OK ){
rc = vec1WriteConfigI64(
pTab, VEC1_CONFIG_BLOCKSIZE_MIN, sqlite3_value_int64(pArg)
);
}
break;
}
default: assert( iCmd==VEC1_CMD_REBUILD ); {
/* TODO: Should this be disallowed if there are active readers? */
if( rc==SQLITE_OK && sqlite3_value_type(pArg)!=SQLITE_NULL ){
u8 aFlat[VEC1_HEADER_SIZE];
Vec1Model mod;
int nByte = 0;
const u8 *aBlob = 0;
sqlite3_stmt *pStmt = 0;
memset(&mod, 0, sizeof(mod));
if( sqlite3_value_type(pArg)==SQLITE_BLOB ){
nByte = sqlite3_value_bytes(pArg);
aBlob = sqlite3_value_blob(pArg);
}else{
Vec1FlatIndex idx = {0, 0};
const char *zText = (const char*)sqlite3_value_text(pArg);
char *zErr = 0;
idx.eIndex = 0;
idx.eDistance = VEC1_DISTANCE_L2;
rc = vec1ParseJsonConfig(
pTab->db, zText, vec1FlatIndexCfg, &idx, &zErr
);
if( rc==SQLITE_OK && idx.eIndex==0 ){
zErr = sqlite3_mprintf("required parameter missing: index");
rc = SQLITE_ERROR;
}
if( zErr!=0 ){
vec1VtabError(pTab, "vec1: %z", zErr);
}
if( rc==SQLITE_OK ){
vec1HeaderWrite(aFlat,
((idx.eIndex==VEC1_INDEX_FLAT) ? VEC1_MODEL_INDEX : 0),
0, 0, 0, idx.eDistance
);
aBlob = aFlat;
nByte = VEC1_HEADER_SIZE;
}
}
/* Decode the model blob here. Just to check if doing so causes any
** errors. If it does, refuse to store it. */
if( rc==SQLITE_OK ){
rc = vec1DecodeModel(aBlob, nByte, &mod, &pTab->base.zErrMsg);
}
if( rc==SQLITE_OK && mod.hdr.nElem>0 ){
if( pTab->cfg.nElem==0 ){
rc = vec1FixVectorSize(pTab, mod.hdr.nElem*sizeof_f32);
}else if( pTab->cfg.nElem!=(int)mod.hdr.nElem ){
vec1VtabError(pTab,
"vec1: model/data vector size mismatch: %d/%d",
(int)mod.hdr.nElem, pTab->cfg.nElem
);
rc = SQLITE_ERROR;
}
}
/* Store the new model in the %_model table. */
if( rc==SQLITE_OK ){
const char zStore[] = "REPLACE INTO %Q.'%q_model' VALUES(1, ?)";
rc = vec1PrepareSql(pTab, &pStmt, zStore, pTab->zDb, pTab->zName);
}
if( rc==SQLITE_OK ){
rc = sqlite3_bind_blob(pStmt, 1, aBlob, nByte, SQLITE_STATIC);
sqlite3_step(pStmt);
vec1StmtFinalize(&rc, pStmt);
}
/* Increment (or create) the model version number in %_config */
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_INCR_CONFIG, &pStmt);
}
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, VEC1_CONFIG_MODEL);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
}
}
if( rc==SQLITE_OK && vec1ModelIsFlat(&pTab->mod) ){
rc = vec1FlatToNone(pTab);
}
if( rc==SQLITE_OK ){
rc = vec1LoadConfig(pTab);
}
if( rc==SQLITE_OK ){
rc = vec1RebuildIndex(pTab, pTab->pTabList->nThread);
}
}
}
}
return rc;
}
/*
** The maximum permitted number of tombstone entries in list containing
** nEntry entries. If this threshold is exceeded, the list is reorganized
** to reduce the number of tombstones to 0.
*/
#define VEC1_TOMBSTONE_THRESHOLD(nEntry) ((nEntry * 2) / 10)
/*
** Locate the index entry for row iRowid, for writing. Array aVec[] is the
** row's transformed vector.
**
** The bucket selected for aVec[] by vec1BestMatchSimple() is searched
** first. If the model has no buckets, this is bucket 0 and it is the only
** bucket searched. Otherwise, if the row is not found there, it may have
** been assigned to a different bucket by code that rounds differently -
** either a different routine, such as the batched routines used by
** 'rebuild', or the same routine on a different platform. In that case
** all other buckets that are near-ties with the selected bucket (see
** vec1BucketIsNearTie()) are searched, followed by all remaining buckets.
**
** If the row is found, (*pLoc) is populated and (*piBucket) is set to
** the bucket it was found in. If it is not found, (*pLoc) is zeroed and
** SQLITE_OK returned. The caller must then treat the row as missing.
*/
static int vec1DeleteFindRowid(
Vec1Tab *pTab, /* Table to search */
const float *aVec, /* Transformed vector */
i64 iRowid, /* Rowid to locate */
int *piBucket, /* OUT: Bucket containing iRowid */
Vec1RowidLocation *pLoc /* OUT: Location of iRowid */
){
const Vec1Model *pMod = &pTab->mod;
const int nElem = (int)pMod->hdr.nElem;
const int nBucket = (int)pMod->hdr.nBucket;
int iBest = 0;
double fBest = 0.0;
int rc = SQLITE_OK;
int iPass;
if( nBucket>0 ){
iBest = vec1BestMatchSimple(pMod->aCentroid, nBucket, aVec, nElem, &fBest);
}
rc = vec1FindByRowid(pTab, iBest, 1, iRowid, pLoc);
if( rc!=SQLITE_OK || pLoc->pList || pLoc->pBlob ){
*piBucket = iBest;
return rc;
}
/* Pass 0 searches buckets that are near-ties with iBest. Pass 1 searches
** all others. */
for(iPass=0; iPass<2; iPass++){
int ii;
for(ii=0; ii<nBucket; ii++){
if( ii!=iBest ){
double fDist;
fDist = vec1L2Dist(aVec, &pMod->aCentroid[ii*nElem], nElem);
if( vec1BucketIsNearTie(fBest, fDist)==(iPass==0) ){
rc = vec1FindByRowid(pTab, ii, 1, iRowid, pLoc);
if( rc!=SQLITE_OK || pLoc->pList || pLoc->pBlob ){
*piBucket = ii;
return rc;
}
}
}
}
}
return SQLITE_OK;
}
/*
** Delete the %_base entry for the row with rowid=iRowid. Also zero-out
** the corresponding %_idx entry.
*/
static int vec1DeleteByRowid(Vec1Tab *pTab, i64 iRowid){
const u8 aTomb4[4] = {0xFF, 0xFF, 0xFF, 0xFF};
const u8 aTomb8[8] = {0, 0, 0, 0, 0, 0, 0, 0};
sqlite3_stmt *pDelete = 0;
int rc = SQLITE_OK;
rc = vec1GetSql(pTab, VEC1_SQL_DEL_LOOKUP_BASE, &pDelete);
if( rc==SQLITE_OK ){
/* Delete the row from the %_base table. Also set variable iBucket to
** the index of the bucket containing the vector. Statement pDelete is
** not reset until the end of this function, as aTransform[] may point
** to the vector it returns. */
sqlite3_bind_int64(pDelete, 1, iRowid);
if( (SQLITE_ROW==sqlite3_step(pDelete))
&& (pTab->mod.hdr.flags & VEC1_MODEL_INDEX)
){
int iBucket = 0;
Vec1RowidLocation loc;
if( pTab->mod.hdr.nCodebook==0 ){
iBucket = sqlite3_column_int(pDelete, 0);
rc = vec1FindByRowid(pTab, iBucket, 1, iRowid, &loc);
}else{
const u8 *aVec = sqlite3_column_blob(pDelete, 0);
int nVec = sqlite3_column_bytes(pDelete, 0);
if( nVec!=(pTab->cfg.nElem*sizeof_f32) ){
rc = VEC1_CORRUPT;
}else{
const float *aTransform = vec1TransformInputVector(
&pTab->mod, pTab->aTmpVec, (const float*)aVec
);
rc = vec1DeleteFindRowid(pTab, aTransform, iRowid, &iBucket, &loc);
}
}
if( rc==SQLITE_OK ){
if( loc.pList==0 && loc.pBlob==0 ){
/* Missing row. This is an error. */
rc = VEC1_CORRUPT;
}else if( (loc.nTombstone+1) > VEC1_TOMBSTONE_THRESHOLD(loc.nEntry) ){
/* This delete would cause the list's tombstone count to exceed
** the threshold. So reorganize the list and remove iRowid in
** one go. */
if( loc.pList ){
rc = vec1ListBuilderCompress(loc.pList, iRowid);
}else{
/* Load the list into memory, compress it so there are no
** tombstones, then write it back to the database. */
Vec1ListBuilder list;
int nVal = sqlite3_blob_bytes(loc.pBlob);
u8 *aVal = sqlite3_malloc(nVal);
memset(&list, 0, sizeof(list));
if( aVal==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_blob_read(loc.pBlob, aVal, nVal, 0);
}
if( rc==SQLITE_OK ){
rc = vec1ListBuilderInit(&list, pTab, iBucket);
}
if( rc==SQLITE_OK ){
/* Initialize iFirst and iLast to 0. vec1ListBuilderCompress()
** will set the correct values. */
rc = vec1ListBuilderLoad(&list, loc.iIdx, 0, 0, aVal, nVal);
}
if( rc==SQLITE_OK ){
rc = vec1ListBuilderCompress(&list, iRowid);
}
if( rc==SQLITE_OK ){
rc = vec1ListBuilderFlush(&list);
}
vec1ListBuilderFree(&list);
sqlite3_free(aVal);
}
}else{
/* Replace the rowid with a tombstone and increment the tombstone
** count. */
const u8 *aTomb = loc.szRowid==4 ? aTomb4 : aTomb8;
if( loc.pList ){
int iOff = loc.iEntry * loc.szRowid;
memcpy(&loc.pList->bufRowid.a[iOff], aTomb, loc.szRowid);
loc.pList->nTombstone++;
}else{
int iOff = VEC1_LIST_SZHDR + (loc.iEntry * loc.szRowid);
rc = sqlite3_blob_write(loc.pBlob, aTomb, loc.szRowid, iOff);
if( rc==SQLITE_OK ){
u8 aTS[4];
vec1PutU32(aTS, loc.nTombstone+1);
rc = sqlite3_blob_write(loc.pBlob, aTS, sizeof(aTS), 8);
}
}
}
if( loc.pBlob ){
int rc2 = sqlite3_blob_close(loc.pBlob);
if( rc==SQLITE_OK ) rc = rc2;
}
}
}
vec1StmtReset(&rc, pDelete);
}
return rc;
}
/*
** xUpdate
**
** argc/argv meanings:
** argc==1 DELETE
** argc>1 && argv[0]==NULL INSERT
** argc>1 && argv[0]!=NULL UPDATE
**
** argv[0] = old rowid (or NULL)
** argv[1] = new rowid (or NULL to auto-assign)
** argv[2+] = column values
*/
static int vec1UpdateMethod(
sqlite3_vtab *pVtab,
int argc,
sqlite3_value **argv,
sqlite3_int64 *pRowid
){
int rc = SQLITE_OK;
Vec1Tab *pTab = (Vec1Tab*)pVtab;
const int iCmd = 2 + VEC1_COLUMN_CMD;
int bDeleteRow = (argc==1); /* Delete row at end of this function */
/* Ensure the configuration has been loaded. */
if( (rc = vec1LoadConfigIfRequired(pTab))!=SQLITE_OK ) return rc;
if( argc>1 ){
int nVec = sqlite3_value_bytes(argv[2+VEC1_COLUMN_VECTOR]);
const void *aVec = sqlite3_value_blob(argv[2+VEC1_COLUMN_VECTOR]);
int eType = sqlite3_value_type(argv[2+VEC1_COLUMN_VECTOR]);
/* Check for a special INSERT. Handle these separately. */
if( sqlite3_value_type(argv[iCmd])!=SQLITE_NULL ){
const char *zCmd = (char*)sqlite3_value_text(argv[iCmd]);
sqlite3_value *pVal = argv[2+VEC1_COLUMN_ARG];
if( sqlite3_value_type(pVal)==SQLITE_NULL ){
pVal = argv[2+VEC1_COLUMN_VECTOR];
}
return vec1SpecialInsert(pTab, zCmd, pVal);
}
/* Check that the supplied vector is the right type and size. If this
** is the first vector written to this table, fix the vector size
** now. */
if( eType!=SQLITE_BLOB ){
rc = vec1VectorTypeError(pTab, eType);
}else if( pTab->cfg.nElem>0 ){
if( nVec!=pTab->cfg.nElem*sizeof_f32 ){
rc = vec1VectorSizeError(pTab, nVec);
}
}else{
rc = vec1FixVectorSize(pTab, nVec);
}
/* Check that the new vector does not contain any NaN or Infinity values */
if( rc==SQLITE_OK ){
rc = vec1VectorIsNonFinite((float*)aVec, pTab->cfg.nElem);
if( rc!=SQLITE_OK ){
vec1VtabError(pTab, "vec1: non-finite value not allowed in vector");
}
}
/* If this is an UPDATE that does not modify the rowid, then delete
** the existing row now. Or, if it is an UPDATE that does modify
** the rowid, schedule deletion of the existing row at the end
** of this function. */
if( rc==SQLITE_OK && sqlite3_value_type(argv[0])==SQLITE_INTEGER ){
if( sqlite3_value_int64(argv[0])==sqlite3_value_int64(argv[1]) ){
rc = vec1DeleteByRowid(pTab, sqlite3_value_int64(argv[0]));
}else{
bDeleteRow = 1;
}
}
/* If this is an INSERT or UPDATE, write the new row. */
if( rc==SQLITE_OK ){
int iBucket = 0;
/* If there is an index (i.e. not index:"none"), then allocate an
** index writer. */
if( rc==SQLITE_OK
&& (pTab->mod.hdr.flags & VEC1_MODEL_INDEX)
&& pTab->pWriter==0
){
rc = vec1WriterAlloc(pTab, 0, &pTab->pWriter);
assert( rc!=SQLITE_OK || pTab->pWriter!=0 );
}
/* Write to the %_base table. */
if( rc==SQLITE_OK ){
sqlite3_stmt *pInsert = 0;
/* If required, quantize the vector. This is because in
** some configurations, the bucket is written into the %_base
** table, so it must be determined now, before the base table entry
** is written */
if( pTab->pWriter ){
Vec1Writer *p = pTab->pWriter;
vec1QuantizeVector(&pTab->mod, p->aResidual, aVec, &iBucket, p->aPQ);
}
*pRowid = 0;
rc = vec1GetSql(pTab, VEC1_SQL_INSERT_BASE, &pInsert);
if( rc==SQLITE_OK ){
int ii;
sqlite3_bind_value(pInsert, 1, argv[1]);
if( vec1ModelIsFlat(&pTab->mod) ){
sqlite3_bind_int(pInsert, 2, iBucket);
}else{
sqlite3_bind_blob(pInsert, 2, aVec, nVec, SQLITE_STATIC);
}
for(ii=0; ii<pTab->nMeta; ii++){
sqlite3_bind_value(pInsert, 3+ii, argv[2+VEC1_COLUMN_VECTOR+1+ii]);
}
sqlite3_step(pInsert);
rc = sqlite3_reset(pInsert);
*pRowid = sqlite3_last_insert_rowid(pTab->db);
}
}
/* If the table has been trained, write the %_idx entry. And, if
** there are any meta-data columns, the %_meta entries as well. */
if( pTab->mod.hdr.flags & VEC1_MODEL_INDEX ){
if( rc==SQLITE_OK ){
rc = vec1WriterVector(pTab->pWriter, *pRowid, aVec, &iBucket);
}
if( rc==SQLITE_OK ){
sqlite3_value **aMeta = &argv[2+VEC1_COLUMN_VECTOR+1];
rc = vec1WriterMetaFromArray(pTab->pWriter, iBucket, aMeta);
}
}
}
}
if( rc==SQLITE_OK && bDeleteRow ){
rc = vec1DeleteByRowid(pTab, sqlite3_value_int64(argv[0]));
}
return rc;
}
static int vec1FinishWriter(Vec1Tab *pTab, int bDiscard){
int rc = SQLITE_OK;
if( bDiscard ){
vec1WriterFinish(pTab->pWriter, SQLITE_ERROR);
}else{
rc = vec1WriterFinish(pTab->pWriter, SQLITE_OK);
}
pTab->pWriter = 0;
return rc;
}
static int vec1SyncMethod(sqlite3_vtab *pVtab){
int rc = SQLITE_OK;
Vec1Tab *pTab = (Vec1Tab*)pVtab;
if( (rc = vec1LoadConfigIfRequired(pTab))!=SQLITE_OK ) return rc;
return vec1FinishWriter(pTab, 0);
}
static int vec1BeginMethod(sqlite3_vtab *pVtab){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
int rc = SQLITE_OK;
rc = vec1LoadConfig(pTab);
return rc;
}
/*
** xRename method for vec1 virtual table. This renames all shadow tables
** and the %_idx_idx index on the %_idx table.
*/
static int vec1RenameMethod(sqlite3_vtab *pVtab, const char *zNew){
const char zFmt[] =
"ALTER TABLE %Q.'%q_config' RENAME TO '%q_config';"
"ALTER TABLE %Q.'%q_base' RENAME TO '%q_base';"
"ALTER TABLE %Q.'%q_idx' RENAME TO '%q_idx';"
"ALTER TABLE %Q.'%q_model' RENAME TO '%q_model';"
"ALTER TABLE %Q.'%q_meta' RENAME TO '%q_meta';"
"UPDATE %Q.sqlite_schema SET name = '%q_idx_idx', sql="
" 'CREATE INDEX ' || quote('%q_idx_idx') || "
" ' ON ' || quote('%q_idx') || '(bucket, first, last)'"
"WHERE type='index' AND name='%q_idx_idx';"
;
Vec1Tab *pTab = (Vec1Tab*)pVtab;/* Virtual table object */
int rc = SQLITE_OK; /* Return value */
int bDefensive = 0; /* Initial value of DEFENSIVE */
int bWritable = 0; /* Initial value of WRITABLE_SCHEMA */
char *zSql = 0; /* SQL script to rename shadow objects */
/* Query the initial values of the DEFENSIVE and WRITABLE_SCHEMA db-config
** options. Then clear DEFENSIVE and set WRITABLE_SCHEMA so that we can
** UPDATE the sqlite_schema entry of the index directly. */
sqlite3_db_config(pTab->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefensive);
sqlite3_db_config(pTab->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, -1, &bWritable);
sqlite3_db_config(pTab->db, SQLITE_DBCONFIG_DEFENSIVE, 0, (int*)0);
sqlite3_db_config(pTab->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, (int*)0);
/* Run the SQL script to rename the shadow tables and index */
zSql = sqlite3_mprintf(zFmt,
pTab->zDb, pTab->zName, zNew, /* %_config */
pTab->zDb, pTab->zName, zNew, /* %_base */
pTab->zDb, pTab->zName, zNew, /* %_idx */
pTab->zDb, pTab->zName, zNew, /* %_model */
pTab->zDb, pTab->zName, zNew, /* %_meta */
pTab->zDb, zNew, zNew, zNew, pTab->zName
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_exec(pTab->db, zSql, 0, 0, &pTab->base.zErrMsg);
sqlite3_free(zSql);
}
/* Restore the values of DEFENSIVE and WRITABLE_SCHEMA */
sqlite3_db_config(pTab->db,SQLITE_DBCONFIG_DEFENSIVE, bDefensive, (int*)0);
sqlite3_db_config(pTab->db,SQLITE_DBCONFIG_WRITABLE_SCHEMA,bWritable,(int*)0);
return rc;
}
static int vec1RollbackMethod(sqlite3_vtab *pVtab){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
vec1UnloadModel(pTab);
return vec1FinishWriter(pTab, 1);
}
static int vec1SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
if( (rc = vec1LoadConfigIfRequired(pTab))!=SQLITE_OK ) return rc;
return vec1FinishWriter(pTab, 0);
}
static int vec1ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
if( (rc = vec1LoadConfigIfRequired(pTab))!=SQLITE_OK ) return rc;
return vec1FinishWriter(pTab, 0);
}
static int vec1RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
Vec1Tab *pTab = (Vec1Tab*)pVtab;
UNUSED_PARAMETER(iSavepoint);
vec1FinishWriter(pTab, 1);
vec1UnloadConfig(pTab);
return SQLITE_OK;
}
/*
** This function is used by the integrity-check code to handle meta-data.
** Argument pBuf contains a meta-data record, complete with VEC1_META_SZHDR
** byte header. When this function is called, (*piOff) contains the offset
** into this buffer of the next meta-data value to read.
**
** When called, this function reads one meta-data value from (*piOff) and
** advances (*piOff) to point to the next value. If pVal is NULL, it returns
** SQLITE_OK at that point. Or, if pVal is not NULL, it is compared to
** the value just read. If they match, SQLITE_OK is returned. Otherwise,
** SQLITE_ERROR.
*/
static int vec1IntegrityMetaValue(
Vec1Buffer *pBuf,
int *piOff,
sqlite3_value *pVal
){
u32 f = vec1GetU32(pBuf->a);
int iOff = *piOff;
Vec1MetaValue val;
int rc = SQLITE_OK;
memset(&val, 0, sizeof(val));
if( (f & VEC1_META_1BYTEINT) ){
val.iVal = pBuf->a[iOff++];
if( val.iVal==VEC1_META_1BYTENULL ){
val.eType = SQLITE_NULL;
}else{
val.eType = SQLITE_INTEGER;
}
}else if( (f & VEC1_META_4BYTEINT) ){
val.iVal = (int)vec1GetU32(&pBuf->a[iOff]);
iOff += sizeof_u32;
if( (u32)val.iVal==VEC1_META_4BYTENULL ){
val.eType = SQLITE_NULL;
}else{
val.eType = SQLITE_INTEGER;
}
}else if( (f & VEC1_META_REAL) ){
u64 v = vec1GetU64(&pBuf->a[iOff]);
iOff += sizeof_f64;
if( v==VEC1_META_REALNULL ){
val.eType = SQLITE_NULL;
}else{
val.eType = SQLITE_FLOAT;
memcpy(&val.fVal, &v, sizeof_f64);
}
}else{
vec1MetaValueRead(pBuf, &iOff, &val);
}
if( rc==SQLITE_OK && pVal ){
int eType = sqlite3_value_type(pVal);
int bMatch = 0;
switch( eType ){
case SQLITE_NULL:
bMatch = (val.eType==SQLITE_NULL);
break;
case SQLITE_INTEGER: {
i64 iVal = sqlite3_value_int64(pVal);
bMatch = (
(val.eType==SQLITE_INTEGER && val.iVal==iVal)
|| (val.eType==SQLITE_FLOAT && val.fVal==(double)iVal)
);
break;
}
case SQLITE_FLOAT: {
double fVal = sqlite3_value_double(pVal);
bMatch = (val.eType==SQLITE_FLOAT && val.fVal==fVal);
break;
}
case SQLITE_TEXT: {
int nVal = sqlite3_value_bytes(pVal);
if( val.eType==SQLITE_TEXT && val.iVal==nVal ){
const char *zVal = (const char*)sqlite3_value_text(pVal);
bMatch = (0==memcmp(val.pPtr, zVal, nVal));
}
break;
}
default: {
int nVal = sqlite3_value_bytes(pVal);
if( val.eType==SQLITE_BLOB && val.iVal==nVal ){
const u8 *zVal = (const u8*)sqlite3_value_blob(pVal);
bMatch = (0==memcmp(val.pPtr, zVal, nVal));
}
assert( eType==SQLITE_BLOB );
break;
}
}
if( bMatch==0 ) rc = SQLITE_ERROR;
}
*piOff = iOff;
return rc;
}
/*
** Return true if the difference between f1 and f2 is smaller than
** (fTol * max(f1, f2)). Or false if they differ by more than this.
*/
static int vec1FloatWithinTolerance(float f1, float f2, float fTol){
if( f1==0.0 || f2==0.0 ) return 0;
if( f2>f1 ){
if( ((f2-f1) / f2) > fTol ) return 0;
}else{
if( ((f1-f2) / f1) > fTol ) return 0;
}
return 1;
}
/*
** Check that the encoded version of aVec[] matches the encoded vector
** stored at entry iEntry of aBlob[]. For BQ encodings, the bit-quantized
** vector must match exactly and the stored norm must be within a small
** tolerance of the calculated value. For PQ encodings, the encoded vectors
** are compared using their reconstruction errors, allowing for small
** differences in floating-point arithmetic.
**
** Return non-zero if the encoded vector is inconsistent with aVec[] or if
** an error occurs. Set *pRc to SQLITE_NOMEM if temporary space cannot be
** allocated.
*/
static int vec1IntegrityCheckEncoded(
int *pRc, /* IN/OUT: Error code */
Vec1Model *pMod, /* Model used to quantize vectors */
const float *aVec, /* Vector to compare quantized entry to */
int iEntry, /* Entry of aBlob[] to compare to */
const u8 *aBlob, /* %_idx.val value */
Vec1Buffer *pBuf /* Buffer to use for temp space */
){
const int nEnc = vec1EncodedVectorSize(pMod);
const int nElem = (int)pMod->hdr.nElem;
/* Allocate the maximum amount of space required by this function -
** enough for one full vector and two encoded vectors. */
int nReq = (nElem * sizeof_f32) + nEnc*2;
if( vec1BufferSize(pBuf, nReq) ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
const int nCodebook = pMod->hdr.nCodebook;
float *v1 = (float*)pBuf->a; /* Space for decoded vector */
u8 *aEnc = (u8*)&v1[nElem]; /* Encoded version of aVec[] */
u8 *aIdx = &aEnc[nEnc]; /* Encoded vector read from aBlob[] */
u32 flags = vec1GetU32(&aBlob[0]);
u32 nEntry = (int)vec1GetU32(&aBlob[4]);
int iOut = 0;
int iOff;
int ii;
/* Encode the vector into aEnc */
vec1EncodeVector(pMod, aVec, aEnc);
/* Read the encoded version of the vector from aBlob[] into aIdx[] */
iOff = nEntry * ((flags & VEC1_LIST_64BIT) ? 8 : 4) + VEC1_LIST_SZHDR;
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ){
int iNorm = iOff + VEC1_LIST_NORM_OFFSET(iEntry, nCodebook);
memcpy(aIdx, &aBlob[iNorm], sizeof_f32);
iOff += VEC1_LIST_CODE_OFFSET(iEntry, nCodebook);
iOut = sizeof_f32;
}else{
iOff += VEC1_LIST_BLOCKED_OFFSET(iEntry, nCodebook);
}
for(ii=0; ii<nCodebook; ii++){
aIdx[iOut++] = aBlob[iOff];
iOff += VEC1_PQ_BLOCKSIZE;
}
if( memcmp(aEnc, aIdx, nEnc) ){
/* The encoded version of the vector does not exactly match the version
** just read from the index. This might be corruption, or it might be
** down to small differences in the floating point arithmetic.
**
** For BQ encodings, we require the body of the encoding to be
** identical, but allow a small tolerance for the norm field.
**
** For PQ encodings, we measure the reconstruction errors of both
** the aEnc[] and aIdx[] encodings. If they are within a small tolerance
** of each other, then both encodings are deemed to be valid. */
const float fTol = 0.000001f;
if( pMod->hdr.flags & VEC1_MODEL_BITQUANT ){
float norm1;
float norm2;
memcpy(&norm1, aEnc, sizeof_f32);
memcpy(&norm2, aIdx, sizeof_f32);
/* Check the norms are close enough. If not, return 1. */
if( vec1FloatWithinTolerance(norm1, norm2, fTol)==0 ) return 1;
/* Check the arrays of bits are identical. If not, return 1. */
if( memcmp(&aEnc[sizeof_f32], &aIdx[sizeof_f32], nEnc-sizeof_f32) ){
return 1;
}
}else{
float e1 = 0.0; /* aEnc[] squared reconstruction error */
float e2 = 0.0; /* aIdx[] squared reconstruction error */
vec1DecodeVector(pMod, aEnc, v1);
e1 = (float)vec1L2Dist(v1, aVec, nElem);
vec1DecodeVector(pMod, aIdx, v1);
e2 = (float)vec1L2Dist(v1, aVec, nElem);
if( vec1FloatWithinTolerance(e1, e2, fTol)==0 ) return 1;
}
}
}
return 0;
}
/*
** Check that a vector with transformed value aVec[] may legitimately be
** stored in bucket iBucket.
**
** If vec1BestMatchSimple() selects bucket iBucket for aVec[], then it
** may. Otherwise, the vector may have been assigned to its bucket by
** different code - either a different routine, such as the batched
** routines used by 'rebuild', or the same routine on a different platform.
** Rounding differences mean such code may select a different bucket when
** aVec[] is (almost) equidistant from two or more centroids. So iBucket
** is also accepted if the distance from aVec[] to centroid iBucket is
** within a small tolerance of the distance to the centroid selected by
** vec1BestMatchSimple().
**
** Return non-zero if aVec[] may not be stored in bucket iBucket, or zero
** otherwise.
*/
static int vec1IntegrityCheckBucket(
const Vec1Model *pMod, /* Model used to quantize vectors */
const float *aVec, /* Transformed vector */
int iBucket /* Bucket the vector is stored in */
){
const int nElem = (int)pMod->hdr.nElem;
const int nBucket = (int)pMod->hdr.nBucket;
double fBest = 0.0;
double fDist = 0.0;
int iBest = 0;
if( nBucket==0 ) return (iBucket!=0);
if( iBucket<0 || iBucket>=nBucket ) return 1;
iBest = vec1BestMatchSimple(pMod->aCentroid, nBucket, aVec, nElem, &fBest);
if( iBest==iBucket ) return 0;
fDist = vec1L2Dist(aVec, &pMod->aCentroid[iBucket*nElem], nElem);
return vec1BucketIsNearTie(fBest, fDist)==0;
}
/*
** The buffer passed as the only argument contains a meta-value array. If
** it is of type GENERIC, test that it is well-formed. Return VEC1_CORRUPT
** if it is not, or SQLITE_OK if it looks ok.
*/
static int vec1IntegrityCheckMetaArray(Vec1Buffer *pBuf){
int rc = SQLITE_OK;
int bGeneric = vec1GetU32(pBuf->a) & VEC1_META_GENERIC;
if( bGeneric ){
u32 nEntry = vec1GetU32(&pBuf->a[4]);
u32 ii;
int iOff = VEC1_META_SZHDR;
for(ii=0; rc==SQLITE_OK && ii<nEntry; ii++){
Vec1MetaValue dummy;
rc = vec1MetaValueRead(pBuf, &iOff, &dummy);
}
}
return rc;
}
/*
** Integrity check method. If the table has been supplied with a model,
** loop through the %_idx table. For each
*/
static int vec1IntegrityMethod(
sqlite3_vtab *pVTab,
const char *zSchema,
const char *zTabName,
int mFlags,
char **pzErr
){
Vec1Tab *pTab = (Vec1Tab*)pVTab;
int rc = SQLITE_OK;
sqlite3_stmt *pScan = 0;
sqlite3_stmt *pLookup = 0;
float *aResidual = 0;
char *zErr = 0;
Vec1Buffer *aBufMeta = 0;
Vec1Buffer tmp = {0, 0, 0};
u8 *aPQ = 0;
UNUSED_PARAMETER2(mFlags, zSchema);
rc = vec1LoadConfig(pTab);
if( rc==SQLITE_OK && (pTab->mod.hdr.flags & VEC1_MODEL_INDEX) ){
int bFlat = vec1ModelIsFlat(&pTab->mod);
Vec1Model *pMod = &pTab->mod;
const int nCodebook = pMod->hdr.nCodebook;
const int nElem = pTab->cfg.nElem;
int *aiOffMeta = 0;
i64 nTotalEntry = 0;
u8 *aIdxPQ = 0;
if( nCodebook>0 && pMod->hdr.nBucket>0 ){
aResidual = vec1MallocZero(pTab->nTmpVec * sizeof_f32);
if( aResidual==0 ){
rc = SQLITE_NOMEM;
goto integrity_failed;
}
}
if( nCodebook>0 ){
aPQ = (u8*)vec1MallocZero((nCodebook+sizeof_f32) * 2);
if( aPQ==0 ){
rc = SQLITE_NOMEM;
goto integrity_failed;
}
aIdxPQ = &aPQ[nCodebook + sizeof_f32];
}
rc = vec1GetSql(pTab, VEC1_SQL_SCAN_IDX, &pScan);
if( rc==SQLITE_OK ){
rc = vec1GetSql(pTab, VEC1_SQL_LOOKUP_BASE, &pLookup);
}
/* If there are one or more meta-value columns, allocate space for
** buffers to load each of them from the database */
if( rc==SQLITE_OK && pTab->nMeta>0 ){
int nByte = pTab->nMeta * (sizeof(Vec1Buffer) + sizeof(int));
aBufMeta = (Vec1Buffer*)vec1MallocZero(nByte);
if( aBufMeta==0 ){
rc = SQLITE_NOMEM;
}else{
aiOffMeta = (int*)&aBufMeta[pTab->nMeta];
}
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
const u8 *aBlob = sqlite3_column_blob(pScan, 0);
int nBlob = sqlite3_column_bytes(pScan, 0);
int iBucket = sqlite3_column_int(pScan, 1);
i64 iId = sqlite3_column_int64(pScan, 2);
int szRowid = 4;
int nEntry = 0;
u32 flags = 0;
int nTombstone = 0;
int ii = 0;
int iMeta = 0;
int nNonTombstone = 0;
if( vec1CheckIdxSize(pTab, aBlob, nBlob) ){
const char *zFmt = "%s: %%_idx entry id=%lld is corrupt";
zErr = sqlite3_mprintf(zFmt, zTabName, sqlite3_column_int64(pScan, 2));
goto integrity_failed;
}
flags = vec1GetU32(&aBlob[0]);
nEntry = (int)vec1GetU32(&aBlob[4]);
nTombstone = (int)vec1GetU32(&aBlob[8]);
if( flags & VEC1_LIST_64BIT ){
szRowid = 8;
}
/* Load the array for each meta-value column from disk */
for(iMeta=0; iMeta<pTab->nMeta; iMeta++){
rc = vec1ReadMeta(pTab, nEntry, &aBufMeta[iMeta], iId, iMeta);
if( rc==SQLITE_OK ){
rc = vec1IntegrityCheckMetaArray(&aBufMeta[iMeta]);
}
if( rc!=SQLITE_OK ){
if( rc==SQLITE_CORRUPT_VTAB ){
const char *zFmt = "%s: error reading meta-list id=%lld,meta=%d";
zErr = sqlite3_mprintf(zFmt, zTabName, iId, iMeta);
rc = SQLITE_OK;
}
goto integrity_failed;
}
aiOffMeta[iMeta] = VEC1_META_SZHDR;
}
for(ii=0; ii<nEntry; ii++){
int iRowidOff = VEC1_LIST_SZHDR + szRowid*ii;
int bExists = 0;
int bTombstone = 0;
i64 iRowid;
if( szRowid==4 ){
iRowid = vec1GetU32(&aBlob[iRowidOff]);
if( iRowid==VEC1_TOMBSTONE_32 ) bTombstone = 1;
}else{
iRowid = vec1GetU64(&aBlob[iRowidOff]);
if( iRowid==VEC1_TOMBSTONE_64 ) bTombstone = 1;
}
if( bTombstone ){
/* Advance the meta-data data iterators 1 value */
for(iMeta=0; iMeta<pTab->nMeta; iMeta++){
vec1IntegrityMetaValue(&aBufMeta[iMeta], &aiOffMeta[iMeta], 0);
}
continue;
}
nNonTombstone++;
sqlite3_bind_int64(pLookup, 1, iRowid);
if( sqlite3_step(pLookup)==SQLITE_ROW ){
bExists = 1;
/* Check that the meta-value arrays contain the correct data */
for(iMeta=0; iMeta<pTab->nMeta; iMeta++){
Vec1Buffer *pBuf = &aBufMeta[iMeta];
sqlite3_value *pVal = sqlite3_column_value(pLookup, 2+iMeta);
if( vec1IntegrityMetaValue(pBuf, &aiOffMeta[iMeta], pVal) ){
const char *zFmt = "%s: meta-value value mismatch for row %lld";
zErr = sqlite3_mprintf(zFmt, zTabName, iRowid);
goto integrity_failed;
}
}
if( bFlat ){
int iBaseBucket = sqlite3_column_int(pLookup, 1);
if( iBaseBucket!=iBucket ){
const char *zFmt =
"%s: bucket value in %%_base row %lld is incorrect";
zErr = sqlite3_mprintf(zFmt, zTabName, iRowid);
goto integrity_failed;
}
}else{
const u8 *aBaseVec = sqlite3_column_blob(pLookup, 1);
const float *aTransform = 0;
int nBaseVec = sqlite3_column_bytes(pLookup, 1);
assert( nCodebook>0 );
/* Check the vector is the right size. */
if( nBaseVec!=nElem*sizeof_f32 ){
const char *zFmt = "%s: vector in %%_base row %lld is wrong size";
zErr = sqlite3_mprintf(zFmt, zTabName, iRowid);
goto integrity_failed;
}
aTransform = vec1TransformInputVector(
&pTab->mod, pTab->aTmpVec, (const float*)aBaseVec
);
/* Check the vector is in the right bucket. */
if( vec1IntegrityCheckBucket(pMod, aTransform, iBucket) ){
const char *zFmt = "%s: vector from row %lld is in wrong bucket";
zErr = sqlite3_mprintf(zFmt, zTabName, iRowid);
goto integrity_failed;
}
/* Check that the contents of the index entry look right. */
{
/* Calculate a PQ code based on the transformed vector. Store
** this in aPQ[]. For binary quantization, aPQ[] contains the
** norm followed by nCodebook bytes of data. For PQ or OPQ,
** aPQ[] just contains the nCodebook bytes of encoded vector. */
const float *aEnc = (const float*)aTransform;
if( (pMod->hdr.flags & VEC1_MODEL_RESIDUAL) ){
const float *aCent = &pMod->aCentroid[iBucket*nElem];
vec1Sub(aResidual, aEnc, aCent, nElem);
aEnc = aResidual;
}
if( vec1IntegrityCheckEncoded(&rc, pMod, aEnc, ii, aBlob, &tmp) ){
const char *zType;
const char *zFmt =
"%s: %%_idx %s does not match calculated %s for row %lld";
zType = (pMod->hdr.flags & VEC1_MODEL_BITQUANT) ? "BQ" : "PQ";
zErr = sqlite3_mprintf(zFmt, zTabName, zType, zType, iRowid);
goto integrity_failed;
}
}
}
}
vec1StmtReset(&rc, pLookup);
if( rc==SQLITE_OK && bExists==0 ){
const char *zFmt = "%s: %%_idx row %lld missing from %%_base";
zErr = sqlite3_mprintf(zFmt, zTabName, iRowid);
goto integrity_failed;
}
}
if( rc==SQLITE_OK && (nNonTombstone+nTombstone)!=nEntry ){
const char *zFmt = "%s: bad tombstone count";
zErr = sqlite3_mprintf(zFmt, zTabName);
goto integrity_failed;
}
nTotalEntry += nNonTombstone;
}
if( rc==SQLITE_OK ){
i64 nGot = -1;
sqlite3_stmt *pStmt = 0;
const char *zCount = "SELECT count(*) FROM %Q.'%q_base'";
rc = vec1PrepareSql(pTab, &pStmt, zCount, pTab->zDb, pTab->zName);
if( rc==SQLITE_OK ){
if( sqlite3_step(pStmt)==SQLITE_ROW ){
nGot = sqlite3_column_int64(pStmt, 0);
}
}
vec1StmtFinalize(&rc, pStmt);
if( rc==SQLITE_OK && nGot!=nTotalEntry ){
const char *zFmt =
"%s: wrong number of entries in %%_base - have %lld, expect %lld";
zErr = sqlite3_mprintf(zFmt, zTabName, nGot, nTotalEntry);
goto integrity_failed;
}
}
}
integrity_failed:
*pzErr = zErr;
sqlite3_free(aResidual);
sqlite3_free(aPQ);
vec1BufferFree(&tmp);
if( aBufMeta ){
int ii;
for(ii=0; ii<pTab->nMeta; ii++){
vec1BufferFree(&aBufMeta[ii]);
}
sqlite3_free(aBufMeta);
}
sqlite3_free(pTab->base.zErrMsg);
pTab->base.zErrMsg = 0;
vec1StmtReset(&rc, pScan);
vec1StmtReset(&rc, pLookup);
return rc;
}
/*************************************************************************
** Start of vec1_cat virtual table.
*/
#define VEC1CAT_SCHEMA \
"CREATE TABLE vec1cat(database, name, model, qprofile)"
typedef struct vec1cat_vtab vec1cat_vtab;
struct vec1cat_vtab {
sqlite3_vtab base;
Vec1TabList *pList;
};
typedef struct vec1cat_cursor vec1cat_cursor;
struct vec1cat_cursor {
sqlite3_vtab_cursor base;
sqlite3_int64 iRowid;
};
static int vec1catConnectMethod(
sqlite3 *db,
void *pAux,
int argc,
const char *const *argv,
sqlite3_vtab **ppVtab,
char **pzErr
){
vec1cat_vtab *pNew;
int rc;
UNUSED_PARAMETER(pzErr);
UNUSED_PARAMETER2(argv, argc);
rc = sqlite3_declare_vtab(db, VEC1CAT_SCHEMA);
if( rc==SQLITE_OK ){
pNew = vec1MallocZero(sizeof(*pNew));
*ppVtab = (sqlite3_vtab*)pNew;
if( pNew==0 ) return SQLITE_NOMEM;
pNew->pList = (Vec1TabList*)pAux;
}
return rc;
}
static int vec1catDisconnectMethod(sqlite3_vtab *pVtab){
vec1cat_vtab *p = (vec1cat_vtab *)pVtab;
sqlite3_free(p);
return SQLITE_OK;
}
static int vec1catOpenMethod(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCur){
vec1cat_cursor *pCur;
UNUSED_PARAMETER(pVtab);
pCur = vec1MallocZero(sizeof(*pCur));
if( pCur==0 ) return SQLITE_NOMEM;
*ppCur = &pCur->base;
return SQLITE_OK;
}
static int vec1catCloseMethod(sqlite3_vtab_cursor *cur){
vec1cat_cursor *pCur = (vec1cat_cursor *)cur;
sqlite3_free(pCur);
return SQLITE_OK;
}
static int vec1catBestIndexMethod(
sqlite3_vtab *pVtab,
sqlite3_index_info *pIdxInfo
){
UNUSED_PARAMETER(pVtab);
pIdxInfo->estimatedCost = (double)10;
pIdxInfo->estimatedRows = 10;
return SQLITE_OK;
}
static int vec1catFilterMethod(
sqlite3_vtab_cursor *cur,
int idxNum,
const char *idxStr,
int argc,
sqlite3_value **argv
){
Vec1TabList *pList = ((vec1cat_vtab*)(cur->pVtab))->pList;
vec1cat_cursor *pCur = (vec1cat_cursor *)cur;
sqlite3_stmt *pSql1 = 0;
int rc = SQLITE_OK;
const char *zSql1 = "SELECT name FROM pragma_database_list";
const char *zSql2 =
"WITH tables(name) AS ("
" SELECT name FROM %Q.sqlite_schema "
" WHERE sql LIKE 'CREATE%%VIRTUAL%%USING%%vec1%%'"
")"
"SELECT * FROM tables CROSS JOIN pragma_table_info(tables.name, %Q)";
UNUSED_PARAMETER2(argc, argv);
UNUSED_PARAMETER2(idxStr, idxNum);
rc = sqlite3_prepare_v2(pList->db, zSql1, -1, &pSql1, 0);
if( rc==SQLITE_OK ){
while( sqlite3_step(pSql1)==SQLITE_ROW ){
const char *zDb = (const char*)sqlite3_column_text(pSql1, 0);
sqlite3_stmt *pSql2 = 0;
char *z = vec1MPrintf(&rc, zSql2, zDb, zDb);
if( rc==SQLITE_OK ){
rc = sqlite3_prepare(pList->db, z, -1, &pSql2, 0);
sqlite3_free(z);
}
if( rc==SQLITE_OK ){
while( sqlite3_step(pSql2)==SQLITE_ROW );
vec1StmtFinalize(&rc, pSql2);
}
}
vec1StmtFinalize(&rc, pSql1);
}
pCur->iRowid = 1;
return rc;
}
static int vec1catNextMethod(sqlite3_vtab_cursor *cur){
vec1cat_cursor *pCur = (vec1cat_cursor *)cur;
pCur->iRowid++;
return SQLITE_OK;
}
static Vec1Tab *vec1catGetTable(vec1cat_cursor *pCur){
Vec1TabList *pList = ((vec1cat_vtab*)(pCur->base.pVtab))->pList;
Vec1Tab *pTab = pList->pFirst;
int ii;
for(ii=1; pTab && ii<pCur->iRowid; ii++){
pTab = pTab->pTabNext;
}
return pTab;
}
static int vec1catEofMethod(sqlite3_vtab_cursor *cur){
return vec1catGetTable((vec1cat_cursor*)cur)==0;
}
/*
** xColumn() method for ephemeral virtual table vec1cat.
*/
static int vec1catColumnMethod(
sqlite3_vtab_cursor *cur,
sqlite3_context *ctx,
int i
){
vec1cat_cursor *pCur = (vec1cat_cursor*)cur;
Vec1Tab *pTab = vec1catGetTable(pCur);
int rc = SQLITE_OK;
switch( i ){
case 0: /* database */
sqlite3_result_text(ctx, pTab->zDb, -1, SQLITE_TRANSIENT);
break;
case 1: /* name */
sqlite3_result_text(ctx, pTab->zName, -1, SQLITE_TRANSIENT);
break;
case 2: { /* model */
Vec1ModelHeader *pHdr = &pTab->mod.hdr;
const char *zDist = 0;
char *zRet = 0;
rc = vec1LoadConfig(pTab);
zDist = ((pHdr->eDistance==VEC1_DISTANCE_L2) ? "l2" : "cos");
if( pHdr->nCodebook==0 && pHdr->nBucket==0 ){
zRet = vec1MPrintf(&rc,
"{\"index\": \"%s\", \"distance\": \"%s\"}",
((pHdr->flags & VEC1_MODEL_INDEX) ? "flat" : "none"), zDist
);
}else{
const char *zQuant = "none";
if( pHdr->nCodebook ){
if( pHdr->flags & VEC1_MODEL_BITQUANT ){
zQuant = "bq";
}else if( pHdr->flags & VEC1_MODEL_ROTATE ){
zQuant = "opq";
}else{
zQuant = "pq";
}
}
zRet = vec1MPrintf(&rc, "{"
"\"index\": \"trained\", "
"\"distance\": \"%s\", "
"\"nbucket\": %d, "
"\"quantizer\": \"%s\", "
"\"codesize\": %d, "
"\"residual\": %s, "
"\"wht\": %s}",
zDist, pHdr->nBucket, zQuant, pHdr->nCodebook,
(pHdr->flags & VEC1_MODEL_RESIDUAL) ? "true" : "false",
(pHdr->flags & VEC1_MODEL_WHT) ? "true" : "false"
);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(ctx, zRet, -1, vec1SqliteFree);
}else{
sqlite3_result_error_code(ctx, rc);
}
break;
}
default: { /* qprofile */
const char *azName[VEC1_QINSTR_NCOUNTER] = {
"total", /* VEC1_QINSTR_TOTAL */
"init", /* VEC1_QINSTR_INITQUERY */
"coarse", /* VEC1_QINSTR_COARSE */
"idxread", /* VEC1_QINSTR_IDXREAD */
"metaread", /* VEC1_QINSTR_METAREAD */
"metascan", /* VEC1_QINSTR_METASCAN */
"lut", /* VEC1_QINSTR_LUT */
"pqscan", /* VEC1_QINSTR_PQSCAN */
"finalsort" /* VEC1_QINSTR_FINALSORT */
};
char *zRet = 0;
int ii;
for(ii=0; ii<VEC1_QINSTR_NCOUNTER; ii++){
zRet = vec1MPrintf(&rc,
"%z%s\n \"%s\": {\"cnt\": %lld, \"cycle\": %lld}",
zRet, (ii==0 ? "" : ","),
azName[ii], pTab->aCount[ii], pTab->aCycle[ii]
);
}
zRet = vec1MPrintf(&rc, "{%z\n}", zRet);
if( rc==SQLITE_OK ){
sqlite3_result_text(ctx, zRet, -1, vec1SqliteFree);
}else{
sqlite3_result_error_code(ctx, rc);
}
}
}
return SQLITE_OK;
}
static int vec1catRowidMethod(
sqlite3_vtab_cursor *cur,
sqlite3_int64 *pRowid
){
vec1cat_cursor *pCur = (vec1cat_cursor *)cur;
*pRowid = pCur->iRowid;
return SQLITE_OK;
}
/*
** End of vec1_cat virtual table.
*************************************************************************/
/*
** Used to get/set per-connection transient parameters. Syntax:
**
** SELECT vec1_config(PARAM)
** SELECT vec1_config(PARAM, VALUE)
**
** There are currently two parameters:
**
** nthread:
** Default number of threads to use for training and rebuilding.
**
** nprobe:
** Default nprobe value for queries that do not specify one.
**
*/
static void vec1ConfigFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **aVal
){
Vec1TabList *pList = (Vec1TabList*)sqlite3_user_data(pCtx);
const char *zParam = (const char*)sqlite3_value_text(aVal[0]);
if( 0==sqlite3_stricmp(zParam, "nthread") ){
if( nVal==2 ){
int nNew = sqlite3_value_int(aVal[1]);
if( nNew<1 || nNew>VEC1_MAX_NTHREAD ){
vec1ResultErrorF(pCtx, "vec1: "
"nthread requires an integer value between 1 and %d",
VEC1_MAX_NTHREAD
);
return;
}
#if VEC1_THREADS
pList->nThread = nNew;
#endif
}
sqlite3_result_int(pCtx, pList->nThread);
}else if( 0==sqlite3_stricmp(zParam, "nprobe") ){
if( nVal==2 ){
int eNumeric = sqlite3_value_numeric_type(aVal[1]);
double fNew = 0.0;
if( eNumeric==SQLITE_INTEGER || eNumeric==SQLITE_FLOAT ){
fNew = sqlite3_value_double(aVal[1]);
}
if( fNew<=0.0 ){
vec1ResultErrorF(pCtx,
"vec1: nprobe must be a numeric value greater than 0.0"
);
return;
}
pList->nProbeArg = fNew;
}
sqlite3_result_double(pCtx, pList->nProbeArg);
}else{
vec1ResultErrorF(pCtx, "vec1: no such parameter: %s", zParam);
}
}
/*
** Destructor for the vec1 module registered using sqlite3_create_module_v2().
** Free the Vec1TabList structure. SQLite should have ensured any virtual
** tables are closed before this is called.
*/
static void vec1ListFree(void *pCtx){
Vec1TabList *pList = (Vec1TabList*)pCtx;
pList->nRef--;
if( pList->nRef==0 ){
assert( pList->pFirst==0 );
sqlite3_free(pList);
}
}
static int initExtension(
sqlite3 *db,
char **pzErr,
const sqlite3_api_routines *pApi
){
static sqlite3_module vec1catModule = {
0, /* iVersion */
0, /* xCreate */
vec1catConnectMethod, /* xConnect */
vec1catBestIndexMethod, /* xBestIndex */
vec1catDisconnectMethod, /* xDisconnect */
0, /* xDestroy */
vec1catOpenMethod, /* xOpen */
vec1catCloseMethod, /* xClose */
vec1catFilterMethod, /* xFilter */
vec1catNextMethod, /* xNext */
vec1catEofMethod, /* xEof */
vec1catColumnMethod, /* xColumn */
vec1catRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
0, /* xIntegrity */
};
static sqlite3_module vec1Module = {
4, /* iVersion */
vec1CreateMethod, /* xCreate */
vec1ConnectMethod, /* xConnect */
vec1BestIndexMethod, /* xBestIndex */
vec1DisconnectMethod, /* xDisconnect */
vec1DestroyMethod, /* xDestroy */
vec1Open, /* xOpen */
vec1Close, /* xClose */
vec1FilterMethod, /* xFilter */
vec1NextMethod, /* xNext */
vec1EofMethod, /* xEof */
vec1ColumnMethod, /* xColumn */
vec1Rowid, /* xRowid */
vec1UpdateMethod, /* xUpdate */
vec1BeginMethod, /* xBegin */
vec1SyncMethod, /* xSync */
0, /* xCommit */
vec1RollbackMethod, /* xRollback */
0, /* xFindFunction */
vec1RenameMethod, /* xRename */
vec1SavepointMethod, /* xSavepoint */
vec1ReleaseMethod, /* xRelease */
vec1RollbackToMethod, /* xRollbackTo */
0, /* xShadowName */
vec1IntegrityMethod /* xIntegrity */
};
static struct Func {
const char *zName;
int nArg;
void (*x)(sqlite3_context*,int,sqlite3_value**);
} aFunc[] = {
{ "vec1_info", 0, vec1InfoFunc },
{ "vec1_to_json", 1, vec1ToJsonFFunc },
{ "vec1_from_json", 1, vec1FromJsonFunc },
{ "vec1_l2_distance", 2, vec1DistanceFuncL2 },
{ "vec1_cos_distance", 2, vec1DistanceFuncCos },
{ "vec1_config", 1, vec1ConfigFunc },
{ "vec1_config", 2, vec1ConfigFunc },
/* Currently undocumented functions. These may become documented when
** datatypes other than 32-bit float are supported for vectors */
{ "vec1_to_json_f", 1, vec1ToJsonFFunc },
{ "vec1_to_json_i", 1, vec1ToJsonIFunc },
};
static struct AggFunc {
const char *zName;
int nArg;
void (*xStep)(sqlite3_context*,int,sqlite3_value**);
void (*xFinal)(sqlite3_context*);
} aAgg[] = {
{ "vec1_train", 1, vec1TrainStep, vec1TrainFinal },
{ "vec1_train", 2, vec1TrainStep, vec1TrainFinal },
};
int ii = 0;
int rc = SQLITE_OK;
Vec1TabList *pList = vec1MallocZero(sizeof(Vec1TabList));
UNUSED_PARAMETER2(pApi, pzErr);
/* Assuming the Vec1TabList object was allocated without error, populate
** it and create the main virtual table module - vec1. */
if( pList==0 ){
rc = SQLITE_NOMEM;
}else{
pList->nThread = 1;
pList->db = db;
pList->nProbeArg = VEC1_QUERY_DEFAULT_NPROBE;
pList->nRef = 1;
rc = sqlite3_create_module_v2(db, "vec1", &vec1Module, pList, vec1ListFree);
}
/* Create the other vtab module - vec1cat */
if( rc==SQLITE_OK ){
pList->nRef++;
rc = sqlite3_create_module_v2(
db, "vec1cat", &vec1catModule, pList, vec1ListFree
);
}
/* Create the scalar functions */
for(ii=0; rc==SQLITE_OK && ii<size_of_array(aFunc); ii++){
rc = sqlite3_create_function(
db, aFunc[ii].zName, aFunc[ii].nArg, SQLITE_UTF8|SQLITE_INNOCUOUS,
(void*)pList, aFunc[ii].x, 0, 0
);
}
/* Create the aggregate functions */
for(ii=0; rc==SQLITE_OK && ii<size_of_array(aAgg); ii++){
struct AggFunc *p = &aAgg[ii];
rc = sqlite3_create_function(
db, p->zName, p->nArg, SQLITE_UTF8|SQLITE_DIRECTONLY, (void*)pList, 0,
p->xStep, p->xFinal
);
}
/* If an error occurred, unregister everything */
if( rc!=SQLITE_OK ){
sqlite3_create_module(db, "vec1", 0, 0);
sqlite3_create_module(db, "vec1cat", 0, 0);
for(ii=0; ii<size_of_array(aFunc); ii++){
const char *z = aFunc[ii].zName;
sqlite3_create_function(db, z, aFunc[ii].nArg, SQLITE_UTF8, 0, 0, 0, 0);
}
for(ii=0; ii<size_of_array(aAgg); ii++){
const char *z = aAgg[ii].zName;
sqlite3_create_function(db, z, aAgg[ii].nArg, SQLITE_UTF8, 0, 0, 0, 0);
}
}
return rc;
}
/**************************************************************************
***************************************************************************
** Below this point gets ugly. Almost all of this is only built for x86
** binaries that support multiple SIMD architectures.
*/
#ifdef VEC1SIMD
/* Arbitrarily selected values for symbols "SCALAR" and "AVX2". */
#define SCALAR 281243987
#define AVX2 949193913
/* Check that VEC1SIMD was set to a recognized value */
# if (VEC1SIMD!=SCALAR && VEC1SIMD!=AVX2)
# error "VEC1SIMD must be defined as either SCALAR or AVX2"
# endif
int sqlite3Vec1InitAVX2(sqlite3*, char**, const sqlite3_api_routines*);
# if VEC1SIMD==AVX2
int sqlite3Vec1InitAVX2(
sqlite3 *db,
char **pzErr,
const sqlite3_api_routines *pApi
){
return initExtension(db, pzErr, pApi);
}
# endif /* VEC1SIMD==AVX2 */
# if VEC1SIMD==SCALAR
# if defined(_MSC_VER)
# include <intrin.h>
static int vec1HasAvx2(void)
{
int regs[4];
__cpuid(regs, 1);
int ecx = regs[2];
if (!(ecx & (1 << 28))) return 0; // AVX
if (!(ecx & (1 << 27))) return 0; // OSXSAVE
unsigned long long xcr0 = _xgetbv(0);
if ((xcr0 & 0x6) != 0x6) return 0;
__cpuidex(regs, 7, 0);
int ebx = regs[1];
return (ebx & (1 << 5)) != 0; // AVX2
}
# else
static int vec1HasAvx2(){
/* Clang/GCC */
__builtin_cpu_init();
return __builtin_cpu_supports("avx2");
}
# endif
static int initMultiExtension(
sqlite3 *db,
char **pzErr,
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
if( vec1HasAvx2() ){
/* Use AVX2 */
rc = sqlite3Vec1InitAVX2(db, pzErr, pApi);
}else{
/* Fall back to scalar. */
rc = initExtension(db, pzErr, pApi);
}
return rc;
}
# define initExtension initMultiExtension
# endif /* VEC1SIMD==SCALAR */
#endif /* defined(VEC1SIMD) */
#if !defined(VEC1SIMD) || VEC1SIMD==SCALAR
#ifndef VEC1_STATIC
SQLITE_EXTENSION_INIT1;
#endif
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_extension_init(
sqlite3 *db,
char **pzErrMsg,
const sqlite3_api_routines *pApi
){
#ifdef SQLITE_EXTENSION_INIT2
SQLITE_EXTENSION_INIT2(pApi);
#endif
return initExtension(db, pzErrMsg, pApi);
}
#ifdef VEC1_STATIC
int sqlite3_vec1_extra_init(const char *z){
sqlite3_auto_extension((void(*)(void))initExtension);
return SQLITE_OK;
}
#endif
#endif /* !defined(VEC1SIMD) || VEC1SIMD==SCALAR */
View Git Blame Copy Permalink