diff --git a/Makefile.in b/Makefile.in index 9451af4202..8910209ce5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -182,7 +182,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ - sqlite3session.lo select.lo status.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1038,6 +1038,9 @@ fts5.c: $(FTS5_SRC) fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c + # Rules to build the 'testfixture' application. # @@ -1144,6 +1147,12 @@ wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo + $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) + +loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la + $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) + # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. diff --git a/Makefile.msc b/Makefile.msc index 001a30be73..b627bee81c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -850,7 +850,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ - sqlite3session.lo select.lo status.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1742,6 +1742,9 @@ fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c + # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test @@ -1868,6 +1871,10 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) + $(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ + $(LDFLAGS) $(LTLINKOPTS) + clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 8df915e955..deba91ae1d 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -73,7 +73,7 @@ AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" - AC_SEARCH_LIBS(pthread_create, pthread) + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi AC_SUBST(THREADSAFE_FLAGS) #----------------------------------------------------------------------- diff --git a/configure b/configure index 850d986306..951b719ed5 100755 --- a/configure +++ b/configure @@ -10464,9 +10464,9 @@ fi if test "$SQLITE_THREADSAFE" = "1"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 -$as_echo_n "checking for library containing pthread_create... " >&6; } -if ${ac_cv_search_pthread_create+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 +$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS @@ -10479,11 +10479,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char pthread_create (); +char pthread_mutexattr_init (); int main () { -return pthread_create (); +return pthread_mutexattr_init (); ; return 0; } @@ -10496,25 +10496,25 @@ for ac_lib in '' pthread; do LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_pthread_create=$ac_res + ac_cv_search_pthread_mutexattr_init=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext - if ${ac_cv_search_pthread_create+:} false; then : + if ${ac_cv_search_pthread_mutexattr_init+:} false; then : break fi done -if ${ac_cv_search_pthread_create+:} false; then : +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : else - ac_cv_search_pthread_create=no + ac_cv_search_pthread_mutexattr_init=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 -$as_echo "$ac_cv_search_pthread_create" >&6; } -ac_res=$ac_cv_search_pthread_create +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 +$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } +ac_res=$ac_cv_search_pthread_mutexattr_init if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" diff --git a/configure.ac b/configure.ac index 0b94d33930..26a60147dd 100644 --- a/configure.ac +++ b/configure.ac @@ -194,7 +194,7 @@ fi AC_SUBST(SQLITE_THREADSAFE) if test "$SQLITE_THREADSAFE" = "1"; then - AC_SEARCH_LIBS(pthread_create, pthread) + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi ########## diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 5f528af793..96ecb38e33 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -84,6 +84,9 @@ struct Fts5PhraseIter { ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer @@ -104,15 +107,29 @@ struct Fts5PhraseIter { ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. The exception is if the table was created +** with the offsets=0 option specified. In this case *piOff is always +** set to -1. +** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** ** xRowid: ** Returns the rowid of the current row. ** @@ -196,7 +213,7 @@ struct Fts5PhraseIter { ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); -** iOff>=0; +** iCol>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol @@ -204,13 +221,51 @@ struct Fts5PhraseIter { ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above -** with the xPhraseFirst() and xPhraseNext() API methods. +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** ** xPhraseNext() ** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 1 */ + int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); @@ -240,8 +295,11 @@ struct Fts5ExtensionApi { int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); - void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 83a71723ff..1daeefc09f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -151,6 +151,7 @@ struct Fts5Config { char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; @@ -179,6 +180,9 @@ struct Fts5Config { #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 @@ -292,6 +296,13 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); + +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ +typedef struct Fts5Termset Fts5Termset; +int sqlite3Fts5TermsetNew(Fts5Termset**); +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); +void sqlite3Fts5TermsetFree(Fts5Termset*); + /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ @@ -328,6 +339,29 @@ int sqlite3Fts5IndexClose(Fts5Index *p); ** } */ +/* +** Return a simple checksum value based on the arguments. +*/ +u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +); + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +); + /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. @@ -413,7 +447,6 @@ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ -u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int); int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); /* @@ -436,6 +469,8 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -492,7 +527,7 @@ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ -int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize); +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); void sqlite3Fts5HashFree(Fts5Hash*); int sqlite3Fts5HashWrite( @@ -629,8 +664,18 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int +); +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); +void sqlite3Fts5ExprClearEof(Fts5Expr*); + int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index e9aab4622a..251a543c5a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -292,3 +292,86 @@ int sqlite3Fts5IsBareword(char t){ } +/************************************************************************* +*/ +typedef struct Fts5TermsetEntry Fts5TermsetEntry; +struct Fts5TermsetEntry { + char *pTerm; + int nTerm; + int iIdx; /* Index (main or aPrefix[] entry) */ + Fts5TermsetEntry *pNext; +}; + +struct Fts5Termset { + Fts5TermsetEntry *apHash[512]; +}; + +int sqlite3Fts5TermsetNew(Fts5Termset **pp){ + int rc = SQLITE_OK; + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); + return rc; +} + +int sqlite3Fts5TermsetAdd( + Fts5Termset *p, + int iIdx, + const char *pTerm, int nTerm, + int *pbPresent +){ + int rc = SQLITE_OK; + *pbPresent = 0; + if( p ){ + int i; + int hash; + Fts5TermsetEntry *pEntry; + + /* Calculate a hash value for this term */ + hash = 104 + iIdx; + for(i=0; iapHash); + + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + pEntry->iIdx = iIdx; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } + } + } + + return rc; +} + +void sqlite3Fts5TermsetFree(Fts5Termset *p){ + if( p ){ + int i; + for(i=0; iapHash); i++){ + Fts5TermsetEntry *pEntry = p->apHash[i]; + while( pEntry ){ + Fts5TermsetEntry *pDel = pEntry; + pEntry = pEntry->pNext; + sqlite3_free(pDel); + } + } + sqlite3_free(p); + } +} + + + diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 6b0e2b28b6..d9778bca20 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -14,7 +14,6 @@ */ - #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 @@ -195,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){ } } + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. @@ -345,6 +371,20 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } @@ -500,6 +540,7 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; + pRet->eDetail = FTS5_DETAIL_FULL; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a747a64c9b..409fbd1d05 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*); struct Fts5Expr { Fts5Index *pIndex; + Fts5Config *pConfig; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ @@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew( }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; + pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; @@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ /* ** Argument pTerm must be a synonym iterator. */ -static int fts5ExprSynonymPoslist( +static int fts5ExprSynonymList( Fts5ExprTerm *pTerm, + int bCollist, Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ @@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist( if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; - i64 dummy; - rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + + if( bCollist ){ + rc = sqlite3Fts5IterCollist(pIter, &a, &n); + }else{ + i64 dummy; + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + } + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( n==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); @@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch( int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist( - pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + rc = fts5ExprSynonymList( + pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); @@ -757,30 +767,51 @@ static int fts5ExprNearTest( ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; - int i; - /* Check that each phrase in the nearset matches the current row. - ** Populate the pPhrase->poslist buffers at the same time. If any - ** phrase is not a match, break out of the loop early. */ - for(i=0; rc==SQLITE_OK && inPhrase; i++){ - Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ - int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); - if( bMatch==0 ) break; - }else{ - rc = sqlite3Fts5IterPoslistBuffer( - pPhrase->aTerm[0].pIter, &pPhrase->poslist - ); + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + int n; + i64 iRowid; + rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + }else if( iRowid==pNode->iRowid && n>0 ){ + pPhrase->poslist.n = 1; + } + } } - } + return pPhrase->poslist.n; + }else{ + int i; - *pRc = rc; - if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ - return 1; - } + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + rc = sqlite3Fts5IterPoslistBuffer( + pPhrase->aTerm[0].pIter, &pPhrase->poslist + ); + } + } - return 0; + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } } static int fts5ExprTokenTest( @@ -1218,6 +1249,9 @@ static int fts5ExprNodeNextMatch( } pNode->bEof = p1->bEof; pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); + } break; } } @@ -1603,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase( if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; @@ -1744,6 +1779,15 @@ void sqlite3Fts5ParseSetColset( Fts5ExprNearset *pNear, Fts5Colset *pColset ){ + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pParse->rc = SQLITE_ERROR; + pParse->zErr = sqlite3_mprintf( + "fts5: column queries are not supported (detail=none)" + ); + sqlite3_free(pColset); + return; + } + if( pNear ){ pNear->pColset = pColset; }else{ @@ -1805,11 +1849,20 @@ Fts5ExprNode *sqlite3Fts5ParseNode( for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 - && pNear->apPhrase[0]->nTerm==1 - && pNear->apPhrase[0]->aTerm[0].pSynonym==0 - ){ - pRet->eType = FTS5_TERM; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ + if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ + pRet->eType = FTS5_TERM; + } + }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + assert( pParse->rc==SQLITE_OK ); + pParse->rc = SQLITE_ERROR; + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_mprintf( + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3_free(pRet); + pRet = 0; } }else{ fts5ExprAddChildren(pRet, pLeft); @@ -1923,6 +1976,9 @@ static char *fts5ExprPrintTcl( for(iTerm=0; zRet && iTermnTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); @@ -2235,3 +2291,230 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ } return nRet; } + +struct Fts5PoslistPopulator { + Fts5PoslistWriter writer; + int bOk; /* True if ok to populate */ + int bMiss; +}; + +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ + Fts5PoslistPopulator *pRet; + pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + if( pRet ){ + int i; + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + if( bLive && + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) + ){ + pRet[i].bMiss = 1; + }else{ + pBuf->n = 0; + } + } + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistPopulator *aPopulator; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +/* +** TODO: Make this more efficient! +*/ +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static int fts5ExprPopulatePoslistsCb( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ +){ + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; + Fts5Expr *pExpr = p->pExpr; + int i; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pTerm; + if( p->aPopulator[i].bOk==0 ) continue; + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + int nTerm = strlen(pTerm->zTerm); + if( (nTerm==nToken || (nTermbPrefix)) + && memcmp(pTerm->zTerm, pToken, nTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff + ); + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistPopulator *aPopulator, + int iCol, + const char *z, int n +){ + int i; + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aPopulator = aPopulator; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + for(i=0; inPhrase; i++){ + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + Fts5Colset *pColset = pNode->pNear->pColset; + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + || aPopulator[i].bMiss + ){ + aPopulator[i].bOk = 0; + }else{ + aPopulator[i].bOk = 1; + } + } + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + if( pNode ){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + break; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + if( bRet==0 ){ + fts5ExprClearPoslists(pNode); + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + break; + } + } + } + return 1; +} + +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + +static void fts5ExprClearEof(Fts5ExprNode *pNode){ + int i; + for(i=0; inChild; i++){ + fts5ExprClearEof(pNode->apChild[i]); + } + pNode->bEof = 0; +} +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ + fts5ExprClearEof(pExpr->pRoot); +} + +/* +** This function is only called for detail=columns tables. +*/ +int sqlite3Fts5ExprPhraseCollist( + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, + int *pnCollist +){ + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + int rc = SQLITE_OK; + + assert( iPhrase>=0 && iPhrasenPhrase ); + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid + && pPhrase->poslist.n>0 + ){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + int bDel = 0; + u8 *a; + rc = fts5ExprSynonymList( + pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist + ); + if( bDel ){ + sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); + *ppCollist = pPhrase->poslist.p; + sqlite3_free(a); + }else{ + *ppCollist = a; + } + }else{ + sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + } + }else{ + *ppCollist = 0; + *pnCollist = 0; + } + + return rc; +} + diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index f184957af6..50ca082711 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -26,6 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; struct Fts5Hash { + int eDetail; /* Copy of Fts5Config.eDetail */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ @@ -62,6 +63,7 @@ struct Fts5HashEntry { int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ + u8 bContent; /* Set content-flag (detail=none mode) */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ @@ -79,7 +81,7 @@ struct Fts5HashEntry { /* ** Allocate a new hash table. */ -int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; @@ -90,6 +92,7 @@ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; + pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; @@ -182,26 +185,46 @@ static int fts5HashResize(Fts5Hash *pHash){ return SQLITE_OK; } -static void fts5HashAddPoslistSize(Fts5HashEntry *p){ +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ if( p->iSzPoslist ){ u8 *pPtr = (u8*)p; - int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ - int nPos = nSz*2 + p->bDel; /* Value of nPos field */ - - assert( p->bDel==0 || p->bDel==1 ); - if( nPos<=127 ){ - pPtr[p->iSzPoslist] = (u8)nPos; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + assert( p->nData==p->iSzPoslist ); + if( p->bDel ){ + pPtr[p->nData++] = 0x00; + if( p->bContent ){ + pPtr[p->nData++] = 0x00; + } + } }else{ - int nByte = sqlite3Fts5GetVarintLen((u32)nPos); - memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); - sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); - p->nData += (nByte-1); + int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ + + assert( p->bDel==0 || p->bDel==1 ); + if( nPos<=127 ){ + pPtr[p->iSzPoslist] = (u8)nPos; + }else{ + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); + p->nData += (nByte-1); + } } - p->bDel = 0; + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; } } +/* +** Add an entry to the in-memory hash table. The key is the concatenation +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). +** +** (bByte || pToken) -> (iRowid,iCol,iPos) +** +** Or, if iCol is negative, then the value is a delete marker. +*/ int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ @@ -214,6 +237,9 @@ int sqlite3Fts5HashWrite( Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); @@ -228,15 +254,18 @@ int sqlite3Fts5HashWrite( /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ + /* Figure out how much space to allocate */ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; + /* Grow the Fts5Hash.aSlot[] array if necessary. */ if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } + /* Allocate new Fts5HashEntry and add it to the hash table. */ p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); @@ -246,70 +275,95 @@ int sqlite3Fts5HashWrite( assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; - p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); - p->iSzPoslist = p->nData; - p->nData += 1; - p->iRowid = iRowid; p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; - nIncr += p->nData; - } - /* Check there is enough space to append a new entry. Worst case scenario - ** is: - ** - ** + 9 bytes for a new rowid, - ** + 4 byte reserved for the "poslist size" varint. - ** + 1 byte for a "new column" byte, - ** + 3 bytes for a new column number (16-bit max) as a varint, - ** + 5 bytes for the new position offset (32-bit max). - */ - if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ - int nNew = p->nAlloc * 2; - Fts5HashEntry *pNew; - Fts5HashEntry **pp; - pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->nAlloc = nNew; - for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); - *pp = pNew; - p = pNew; + /* Add the first rowid field to the hash-entry */ + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); + p->iRowid = iRowid; + + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + } + + nIncr += p->nData; + }else{ + + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario + ** is: + ** + ** + 9 bytes for a new rowid, + ** + 4 byte reserved for the "poslist size" varint. + ** + 1 byte for a "new column" byte, + ** + 3 bytes for a new column number (16-bit max) as a varint, + ** + 5 bytes for the new position offset (32-bit max). + */ + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ + int nNew = p->nAlloc * 2; + Fts5HashEntry *pNew; + Fts5HashEntry **pp; + pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->nAlloc = nNew; + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); + *pp = pNew; + p = pNew; + } + nIncr -= p->nData; } + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); + pPtr = (u8*)p; - nIncr -= p->nData; /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); - p->iSzPoslist = p->nData; - p->nData += 1; - p->iCol = 0; - p->iPos = 0; p->iRowid = iRowid; + bNew = 1; + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + p->iPos = 0; + } } if( iCol>=0 ){ - /* Append a new column value, if necessary */ - assert( iCol>=p->iCol ); - if( iCol!=p->iCol ){ - pPtr[p->nData++] = 0x01; - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); - p->iCol = iCol; - p->iPos = 0; - } + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + p->bContent = 1; + }else{ + /* Append a new column value, if necessary */ + assert( iCol>=p->iCol ); + if( iCol!=p->iCol ){ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = iCol; + p->iPos = 0; + }else{ + bNew = 1; + p->iCol = iPos = iCol; + } + } - /* Append the new position offset */ - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); - p->iPos = iPos; + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } + } }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; } - nIncr += p->nData; + nIncr += p->nData; *pHash->pnByte += nIncr; return SQLITE_OK; } @@ -423,7 +477,7 @@ int sqlite3Fts5HashQuery( } if( p ){ - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ @@ -459,7 +513,7 @@ void sqlite3Fts5HashScanEntry( Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c11abda5ba..2f5b3bdaae 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -433,6 +433,9 @@ struct Fts5SegIter { Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ + /* Next method */ + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); + /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; @@ -452,7 +455,7 @@ struct Fts5SegIter { Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ - int bDel; /* True if the delete flag is set */ + u8 bDel; /* True if the delete flag is set */ }; /* @@ -466,7 +469,6 @@ struct Fts5SegIter { #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 - /* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or @@ -1492,13 +1494,29 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ - int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); - fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); - pIter->bDel = (nSz & 0x0001); - pIter->nPos = nSz>>1; + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); + pIter->bDel = 0; + pIter->nPos = 1; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->bDel = 1; + iOff++; + if( iOffpLeaf->p[iOff]==0 ){ + pIter->nPos = 1; + iOff++; + }else{ + pIter->nPos = 0; + } + } + }else{ + int nSz; + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } pIter->iLeafOffset = iOff; - assert_nc( pIter->nPos>=0 ); } } @@ -1559,6 +1577,20 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ fts5SegIterLoadRowid(p, pIter); } +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); + +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + pIter->xNext = fts5SegIterNext_Reverse; + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xNext = fts5SegIterNext_None; + }else{ + pIter->xNext = fts5SegIterNext; + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -1584,6 +1616,7 @@ static void fts5SegIterInit( if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); + fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); @@ -1615,6 +1648,7 @@ static void fts5SegIterInit( ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; @@ -1627,15 +1661,24 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; - int nPos; - int bDummy; - i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); - i += nPos; + if( eDetail==FTS5_DETAIL_NONE ){ + /* todo */ + if( i=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); pIter->iRowid += iDelta; + /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); @@ -1714,6 +1757,108 @@ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used by reverse iterators. +*/ +static void fts5SegIterNext_Reverse( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->pNextLeaf==0 ); + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + i64 iDelta; + + pIter->iRowidOffset--; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += pIter->nPos; + } + fts5GetVarint(&a[iOff], (u64*)&iDelta); + pIter->iRowid -= iDelta; + }else{ + fts5SegIterReverseNewPage(p, pIter); + } +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used if detail=none and the +** iterator is not a reverse direction iterator. +*/ +static void fts5SegIterNext_None( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + int iOff; + + assert( p->rc==SQLITE_OK ); + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); + + ASSERT_SZLEAF_OK(pIter->pLeaf); + iOff = pIter->iLeafOffset; + + /* Next entry is on the next page */ + if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( p->rc || pIter->pLeaf==0 ) return; + pIter->iRowid = 0; + iOff = 4; + } + + if( iOffiEndofDoclist ){ + /* Next entry is on the current page */ + i64 iDelta; + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); + pIter->iLeafOffset = iOff; + pIter->iRowid += iDelta; + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ + if( pIter->pSeg ){ + int nKeep = 0; + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); + } + pIter->iLeafOffset = iOff; + fts5SegIterLoadTerm(p, pIter, nKeep); + }else{ + const u8 *pList = 0; + const char *zTerm = 0; + int nList; + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + if( pList==0 ) goto next_none_eof; + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList; + sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + } + + if( pbNewTerm ) *pbNewTerm = 1; + }else{ + goto next_none_eof; + } + + fts5SegIterLoadNPos(p, pIter); + + return; + next_none_eof: + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + + /* ** Advance iterator pIter to the next entry. ** @@ -1726,141 +1871,130 @@ static void fts5SegIterNext( Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ - assert( pbNewTerm==0 || *pbNewTerm==0 ); - if( p->rc==SQLITE_OK ){ - if( pIter->flags & FTS5_SEGITER_REVERSE ){ - assert( pIter->pNextLeaf==0 ); - if( pIter->iRowidOffset>0 ){ - u8 *a = pIter->pLeaf->p; - int iOff; - int nPos; - int bDummy; - i64 iDelta; + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; - pIter->iRowidOffset--; - pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; - iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); - iOff += nPos; - fts5GetVarint(&a[iOff], (u64*)&iDelta); - pIter->iRowid -= iDelta; - fts5SegIterLoadNPos(p, pIter); - }else{ - fts5SegIterReverseNewPage(p, pIter); + assert( pbNewTerm==0 || *pbNewTerm==0 ); + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + /* Search for the end of the position list within the current page. */ + u8 *a = pLeaf->p; + int n = pLeaf->szLeaf; + + ASSERT_SZLEAF_OK(pLeaf); + iOff = pIter->iLeafOffset + pIter->nPos; + + if( iOffiEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ + bNewTerm = 1; + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); } }else{ - Fts5Data *pLeaf = pIter->pLeaf; - int iOff; - int bNewTerm = 0; - int nKeep = 0; - - /* Search for the end of the position list within the current page. */ - u8 *a = pLeaf->p; - int n = pLeaf->szLeaf; + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); + } + pIter->iLeafOffset = iOff; + }else if( pIter->pSeg==0 ){ + const u8 *pList = 0; + const char *zTerm = 0; + int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), + (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; + } + }else{ + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; ASSERT_SZLEAF_OK(pLeaf); - iOff = pIter->iLeafOffset + pIter->nPos; - - if( iOffiEndofDoclist ); - if( iOff>=pIter->iEndofDoclist ){ - bNewTerm = 1; - if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ - iOff += fts5GetVarint32(&a[iOff], nKeep); - } - }else{ - u64 iDelta; - iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); - pIter->iRowid += iDelta; - assert_nc( iDelta>0 ); - } + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; - }else if( pIter->pSeg==0 ){ - const u8 *pList = 0; - const char *zTerm = 0; - int nList = 0; - assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); - if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ - sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); - } - if( pList==0 ){ - fts5DataRelease(pIter->pLeaf); - pIter->pLeaf = 0; - }else{ - pIter->pLeaf->p = (u8*)pList; - pIter->pLeaf->nn = nList; - pIter->pLeaf->szLeaf = nList; - pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); - pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); - *pbNewTerm = 1; - } - }else{ - iOff = 0; - /* Next entry is not on the current page */ - while( iOff==0 ){ - fts5SegIterNextPage(p, pIter); - pLeaf = pIter->pLeaf; - if( pLeaf==0 ) break; - ASSERT_SZLEAF_OK(pLeaf); - if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ - iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - - if( pLeaf->nn>pLeaf->szLeaf ){ - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( - &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist ); - } + } - } - else if( pLeaf->nn>pLeaf->szLeaf ){ - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( - &pLeaf->p[pLeaf->szLeaf], iOff + } + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff ); - pIter->iLeafOffset = iOff; - pIter->iEndofDoclist = iOff; - bNewTerm = 1; - } - if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - } + pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; + bNewTerm = 1; } + assert_nc( iOffszLeaf ); + if( iOff>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + } + } - /* Check if the iterator is now at EOF. If so, return early. */ - if( pIter->pLeaf ){ - if( bNewTerm ){ - if( pIter->flags & FTS5_SEGITER_ONETERM ){ - fts5DataRelease(pIter->pLeaf); - pIter->pLeaf = 0; - }else{ - fts5SegIterLoadTerm(p, pIter, nKeep); - fts5SegIterLoadNPos(p, pIter); - if( pbNewTerm ) *pbNewTerm = 1; - } - }else{ - /* The following could be done by calling fts5SegIterLoadNPos(). But - ** this block is particularly performance critical, so equivalent - ** code is inlined. */ - int nSz; - assert( p->rc==SQLITE_OK ); - fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); - pIter->bDel = (nSz & 0x0001); - pIter->nPos = nSz>>1; - assert_nc( pIter->nPos>=0 ); - } + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf ){ + if( bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); + fts5SegIterLoadNPos(p, pIter); + if( pbNewTerm ) *pbNewTerm = 1; } + }else{ + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. + ** + ** Later: Switched back to fts5SegIterLoadNPos() because it supports + ** detail=none mode. Not ideal. + */ + int nSz; + assert( p->rc==SQLITE_OK ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOffiLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==( + pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) + )); + pIter->iLeafOffset = iPoslist; /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to @@ -1965,11 +2109,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#define fts5IndexSkipVarint(a, iOff) { \ - int iEnd = iOff+9; \ - while( (a[iOff++] & 0x80) && iOffnn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); - pIter->iEndofDoclist = pLeaf->nn+1; + pIter->iEndofDoclist = pLeaf->nn; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; @@ -2238,6 +2379,8 @@ static void fts5SegIterHashInit( fts5SegIterLoadNPos(p, pIter); } } + + fts5SegIterSetNext(p, pIter); } /* @@ -2481,7 +2624,7 @@ static void fts5SegIterNextFrom( } do{ - if( bMove ) fts5SegIterNext(p, pIter, 0); + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; @@ -2515,7 +2658,9 @@ static void fts5MultiIterAdvanced( for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ - fts5SegIterNext(p, &pIter->aSeg[iEq], 0); + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, 0); i = pIter->nSeg + iEq; } } @@ -2602,7 +2747,7 @@ static void fts5MultiIterNext( if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ - fts5SegIterNext(p, pSeg, &bNewTerm); + pSeg->xNext(p, pSeg, &bNewTerm); } if( pSeg->pLeaf==0 || bNewTerm @@ -2630,7 +2775,8 @@ static void fts5MultiIterNext2( Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; - fts5SegIterNext(p, pSeg, &bNewTerm); + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ @@ -2750,7 +2896,8 @@ static void fts5MultiIterNew( for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - fts5SegIterNext(p, &pNew->aSeg[iEq], 0); + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); fts5MultiIterAdvanced(p, pNew, iEq, iIter); } } @@ -2800,6 +2947,7 @@ static void fts5MultiIterNew2( }else{ pNew->bEof = 1; } + fts5SegIterSetNext(p, pIter); *ppOut = pNew; } @@ -2869,6 +3017,9 @@ static void fts5ChunkIterate( int pgno = pSeg->iLeafPgno; int pgnoSave = 0; + /* This function does notmwork with detail=none databases. */ + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } @@ -3292,8 +3443,7 @@ static void fts5WriteAppendTerm( static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, - i64 iRowid, - int nPos + i64 iRowid ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; @@ -3320,8 +3470,6 @@ static void fts5WriteAppendRowid( pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; - - fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); } } @@ -3517,6 +3665,7 @@ static void fts5IndexMergeLevel( Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ + int eDetail = p->pConfig->eDetail; assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); @@ -3586,11 +3735,21 @@ static void fts5IndexMergeLevel( /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ - nPos = pSegIter->nPos*2 + pSegIter->bDel; - fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos); + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); - /* Append the position-list data to the output */ - fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + if( eDetail==FTS5_DETAIL_NONE ){ + if( pSegIter->bDel ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + if( pSegIter->nPos>0 ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + } + } + }else{ + /* Append the position-list data to the output */ + nPos = pSegIter->nPos*2 + pSegIter->bDel; + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + } } /* Flush the last leaf page to disk. Set the output segment b-tree height @@ -3778,7 +3937,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( iSegid ){ const int pgsz = p->pConfig->pgsz; - + int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -3821,12 +3980,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffn<=pBuf->nSpace ); - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); - }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffp[pBuf->n++] = 0; + iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - if( iPos>=nCopy ) break; } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; } - iOff += nCopy; } } @@ -4001,6 +4173,14 @@ struct PoslistCallbackCtx { int eState; /* See above */ }; +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + /* ** TODO: Make this more efficient! */ @@ -4012,6 +4192,28 @@ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +static void fts5PoslistOffsetsCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } + } + } +} + static void fts5PoslistFilterCallback( Fts5Index *p, void *pContext, @@ -4079,12 +4281,20 @@ static void fts5SegiterPoslist( if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ - PoslistCallbackCtx sCtx; - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - sCtx.eState = fts5IndexColsetTest(pColset, 0); - assert( sCtx.eState==0 || sCtx.eState==1 ); - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } } } } @@ -4127,6 +4337,16 @@ static int fts5IndexExtractCol( return p - (*pa); } +static int fts5AppendRowid( + Fts5Index *p, + i64 iDelta, + Fts5IndexIter *pMulti, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); + return 0; +} /* ** Iterator pMulti currently points to a valid entry (not EOF). This @@ -4154,8 +4374,8 @@ static int fts5AppendPoslist( assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + if( p->pConfig->eDetail==FTS5_DETAIL_FULL + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; @@ -4200,13 +4420,13 @@ static int fts5AppendPoslist( } } } - } } return 0; } + static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; @@ -4267,6 +4487,69 @@ static void fts5MergeAppendDocid( (iLastRowid) = (iRowid); \ } +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ + Fts5Buffer tmp = *p1; + *p1 = *p2; + *p2 = tmp; +} + +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ + int i = *piOff; + if( i>=pBuf->n ){ + *piOff = -1; + }else{ + u64 iVal; + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); + *piRowid += iVal; + } +} + +/* +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. +** In this case the buffers consist of a delta-encoded list of rowids only. +*/ +static void fts5MergeRowidLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + Fts5Buffer *p2 /* Second list to merge */ +){ + int i1 = 0; + int i2 = 0; + i64 iRowid1 = 0; + i64 iRowid2 = 0; + i64 iOut = 0; + + Fts5Buffer out; + memset(&out, 0, sizeof(out)); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); + if( p->rc ) return; + + fts5NextRowid(p1, &i1, &iRowid1); + fts5NextRowid(p2, &i2, &iRowid2); + while( i1>=0 || i2>=0 ){ + if( i1>=0 && (i2<0 || iRowid1iOut ); + fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); + iOut = iRowid1; + fts5NextRowid(p1, &i1, &iRowid1); + }else{ + assert( iOut==0 || iRowid2>iOut ); + fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); + iOut = iRowid2; + if( i1>=0 && iRowid1==iRowid2 ){ + fts5NextRowid(p1, &i1, &iRowid1); + } + fts5NextRowid(p2, &i2, &iRowid2); + } + } + + fts5BufferSwap(&out, p1); + fts5BufferFree(&out); +} + /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before @@ -4335,7 +4618,9 @@ static void fts5MergePrefixLists( sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); } } - p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); + if( iNew!=writer.iPrev || tmp.n==0 ){ + p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); + } } /* WRITEPOSLISTSIZE */ @@ -4352,12 +4637,6 @@ static void fts5MergePrefixLists( } } -static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ - Fts5Buffer tmp = *p1; - *p1 = *p2; - *p2 = tmp; -} - static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -4370,6 +4649,16 @@ static void fts5SetupPrefixIter( Fts5Buffer *aBuf; const int nBuf = 32; + void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); + int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + xMerge = fts5MergeRowidLists; + xAppend = fts5AppendRowid; + }else{ + xMerge = fts5MergePrefixLists; + xAppend = fts5AppendPoslist; + } + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); @@ -4402,21 +4691,21 @@ static void fts5SetupPrefixIter( fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ - fts5MergePrefixLists(p, &doclist, &aBuf[i]); + xMerge(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } iLastRowid = 0; } - if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ + if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ iLastRowid = iRowid; } } for(i=0; irc==SQLITE_OK ){ - fts5MergePrefixLists(p, &doclist, &aBuf[i]); + xMerge(p, &doclist, &aBuf[i]); } fts5BufferFree(&aBuf[i]); } @@ -4446,7 +4735,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ - p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ @@ -4567,7 +4856,11 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ -static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){ +int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +){ int n = 0; int i; for(i=0; inPrefix && rc==SQLITE_OK; i++){ - int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); + const int nChar = pConfig->aPrefix[i]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, @@ -4802,9 +5096,16 @@ int sqlite3Fts5IterPoslist( i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int eDetail = pIter->pIndex->pConfig->eDetail; + assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + if( eDetail==FTS5_DETAIL_NONE ){ + *pn = pSeg->nPos; + }else + if( eDetail==FTS5_DETAIL_FULL + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset==0 || pIter->bFiltered ){ *pn = pSeg->nPos; @@ -4821,12 +5122,25 @@ int sqlite3Fts5IterPoslist( }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - *pp = pIter->poslist.p; + if( eDetail==FTS5_DETAIL_FULL ){ + *pp = pIter->poslist.p; + } *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); } +int sqlite3Fts5IterCollist( + Fts5IndexIter *pIter, + const u8 **pp, /* OUT: Pointer to position-list data */ + int *pn /* OUT: Size of position-list in bytes */ +){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + *pp = pIter->poslist.p; + *pn = pIter->poslist.n; + return SQLITE_OK; +} + /* ** This function is similar to sqlite3Fts5IterPoslist(), except that it ** copies the position list into the buffer supplied as the second @@ -4940,7 +5254,7 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ /* ** Return a simple checksum value based on the arguments. */ -static u64 fts5IndexEntryCksum( +u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, @@ -5010,30 +5324,37 @@ static int fts5QueryCksum( int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ + int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; + Fts5Buffer buf = {0, 0, 0}; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ - i64 dummy; - const u8 *pPos; - int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); - if( rc==SQLITE_OK ){ - Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); - sReader.bEof==0; - sqlite3Fts5PoslistReaderNext(&sReader) - ){ - int iCol = FTS5_POS2COLUMN(sReader.iPos); - int iOff = FTS5_POS2OFFSET(sReader.iPos); - cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + + if( eDetail==FTS5_DETAIL_NONE ){ + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); + }else{ + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); + if( rc==SQLITE_OK ){ + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + } } + } + if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNext(pIdxIter); } } sqlite3Fts5IterClose(pIdxIter); + fts5BufferFree(&buf); *pCksum = cksum; return rc; @@ -5327,7 +5648,7 @@ static void fts5IndexIntegrityCheckSegment( /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums -** as calculated by fts5IndexEntryCksum() is cksum. +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without @@ -5335,6 +5656,7 @@ static void fts5IndexIntegrityCheckSegment( ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ + int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5IndexIter *pIter; /* Used to iterate through entire index */ @@ -5386,12 +5708,18 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); - poslist.n = 0; - fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist); - while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ - int iCol = FTS5_POS2COLUMN(iPos); - int iTokOff = FTS5_POS2OFFSET(iPos); - cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + if( eDetail==FTS5_DETAIL_NONE ){ + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } + }else{ + poslist.n = 0; + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); @@ -5407,34 +5735,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ return fts5IndexReturn(p); } - -/* -** Calculate and return a checksum that is the XOR of the index entry -** checksum of all entries that would be generated by the token specified -** by the final 5 arguments. -*/ -u64 sqlite3Fts5IndexCksum( - Fts5Config *pConfig, /* Configuration object */ - i64 iRowid, /* Document term appears in */ - int iCol, /* Column term appears in */ - int iPos, /* Position term appears in */ - const char *pTerm, int nTerm /* Term at iPos */ -){ - u64 ret = 0; /* Return value */ - int iIdx; /* For iterating through indexes */ - - ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); - - for(iIdx=0; iIdxnPrefix; iIdx++){ - int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); - if( nByte ){ - ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); - } - } - - return ret; -} - /************************************************************************* ************************************************************************** ** Below this point is the implementation of the fts5_decode() scalar @@ -5792,4 +6092,3 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ } return rc; } - diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 70912de4f8..fec589efcb 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -226,6 +226,7 @@ struct Fts5Cursor { #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) @@ -639,6 +640,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){ FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST ); } @@ -721,15 +723,18 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); - for(i=0; i<(pSorter->nIdx-1); i++){ - int iVal; - a += fts5GetVarint32(a, iVal); - iOff += iVal; - pSorter->aIdx[i] = iOff; + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; } - pSorter->aIdx[i] = &aBlob[nBlob] - a; - pSorter->aPoslist = a; fts5CsrNewrow(pCsr); } @@ -1167,6 +1172,7 @@ static int fts5FilterMethod( pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); + sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; @@ -1596,6 +1602,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ return rc; } +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); + static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; @@ -1645,17 +1653,72 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } -static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ - int n; - if( pCsr->pSorter ){ +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + } + } + return rc; +} + +static int fts5CsrPoslist( + Fts5Cursor *pCsr, + int iPhrase, + const u8 **pa, + int *pn +){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + int rc = SQLITE_OK; + int bLive = (pCsr->pSorter==0); + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistPopulator *aPopulator; + int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); + if( aPopulator==0 ) rc = SQLITE_NOMEM; + for(i=0; inCol && rc==SQLITE_OK; i++){ + int n; const char *z; + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aPopulator, i, z, n + ); + } + } + sqlite3_free(aPopulator); + + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - n = pSorter->aIdx[iPhrase] - i1; + *pn = pSorter->aIdx[iPhrase] - i1; *pa = &pSorter->aPoslist[i1]; }else{ - n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); } - return n; + + return rc; } /* @@ -1680,43 +1743,46 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int i; /* Initialize all iterators */ - for(i=0; i=pCsr->nInstAlloc ){ - pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; - aInst = (int*)sqlite3_realloc( - pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 - ); - if( aInst ){ - pCsr->aInst = aInst; - }else{ - rc = SQLITE_NOMEM; - break; + nInst++; + if( nInst>=pCsr->nInstAlloc ){ + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc( + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + }else{ + rc = SQLITE_NOMEM; + break; + } } - } - aInst = &pCsr->aInst[3 * (nInst-1)]; - aInst[0] = iBest; - aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); - aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } } pCsr->nInstCount = nInst; @@ -1749,6 +1815,12 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; +#if 0 + }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 2]; + *piOff = -1; +#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -1762,27 +1834,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } -static int fts5ApiColumnText( - Fts5Context *pCtx, - int iCol, - const char **pz, - int *pn -){ - int rc = SQLITE_OK; - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ - *pz = 0; - *pn = 0; - }else{ - rc = fts5SeekCursor(pCsr, 0); - if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); - } - } - return rc; -} - static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, @@ -1927,20 +1978,91 @@ static void fts5ApiPhraseNext( } } -static void fts5ApiPhraseFirst( +static int fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); - pIter->b = &pIter->a[n]; - *piCol = 0; - *piOff = 0; - fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + int n; + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + } + return rc; } +static void fts5ApiPhraseNextColumn( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + }else{ + int iIncr; + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); + *piCol += (iIncr-2); + } + }else{ + while( 1 ){ + int dummy; + if( pIter->a>=pIter->b ){ + *piCol = -1; + return; + } + if( pIter->a[0]==0x01 ) break; + pIter->a += fts5GetVarint32(pIter->a, dummy); + } + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + } +} + +static int fts5ApiPhraseFirstColumn( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + int n; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); + } + }else{ + int n; + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } + } + } + + return rc; +} + + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); @@ -1964,9 +2086,10 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, + fts5ApiPhraseFirstColumn, + fts5ApiPhraseNextColumn, }; - /* ** Implementation of API function xQueryPhrase(). */ @@ -2098,20 +2221,46 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: - /* Append the varints */ - for(i=0; i<(nPhrase-1); i++){ - const u8 *dummy; - int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); - sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); - } + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } - /* Append the position lists */ - for(i=0; ipExpr, i, &pPoslist); - sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + /* Append the position lists */ + for(i=0; ipExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && ipExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 13297c1509..4a1ce3a3d9 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -825,14 +825,16 @@ struct Fts5IntegrityCtx { int iCol; int szCol; u64 cksum; + Fts5Termset *pTermset; Fts5Config *pConfig; }; + /* ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( - void *pContext, /* Pointer to Fts5InsertCtx object */ + void *pContext, /* Pointer to Fts5IntegrityCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ @@ -840,13 +842,56 @@ static int fts5StorageIntegrityCallback( int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; + Fts5Termset *pTermset = pCtx->pTermset; + int bPresent; + int ii; + int rc = SQLITE_OK; + int iPos; + int iCol; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } - pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken - ); - return SQLITE_OK; + + switch( pCtx->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + iPos = pCtx->szCol-1; + iCol = pCtx->iCol; + break; + + case FTS5_DETAIL_COLUMNS: + iPos = pCtx->iCol; + iCol = 0; + break; + + default: + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); + iPos = 0; + iCol = 0; + break; + } + + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, 0, pToken, nToken + ); + } + + for(ii=0; rc==SQLITE_OK && iipConfig->nPrefix; ii++){ + const int nChar = pCtx->pConfig->aPrefix[ii]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); + if( bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte + ); + } + } + } + + return rc; } /* @@ -882,22 +927,37 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_column_text(pScan, i+1), - sqlite3_column_bytes(pScan, i+1), - (void*)&ctx, - fts5StorageIntegrityCallback - ); - if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + (const char*)sqlite3_column_text(pScan, i+1), + sqlite3_column_bytes(pScan, i+1), + (void*)&ctx, + fts5StorageIntegrityCallback + ); + } + if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + if( rc!=SQLITE_OK ) break; } rc2 = sqlite3_reset(pScan); diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index b470c557d3..4a331a59f4 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -235,6 +235,8 @@ static int xF5tApi( { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ + { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ + { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ { 0, 0, 0} }; @@ -431,6 +433,62 @@ static int xF5tApi( break; } + CASE(16, "xPhraseForeach") { + int iPhrase; + int iCol; + int iOff; + const char *zColvar; + const char *zOffvar; + Tcl_Obj *pScript = objv[5]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + zOffvar = Tcl_GetString(objv[4]); + + for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); + iCol>=0; + p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) + ){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + + CASE(17, "xPhraseColumnForeach") { + int iPhrase; + int iCol; + const char *zColvar; + Tcl_Obj *pScript = objv[4]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + + for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); + iCol>=0; + p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol) + ){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 355f23330d..28331773c0 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -134,7 +134,7 @@ static int fts5MatchinfoXCb( int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); - iOff>=0; + iCol>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; @@ -211,18 +211,31 @@ static int fts5MatchinfoLocalCb( int rc = SQLITE_OK; switch( f ){ - case 'b': + case 'b': { + int iPhrase; + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; inPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iCol; + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); + iCol>=0; + pApi->xPhraseNextColumn(pFts, &iter, &iCol) + ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + } + } + + break; + } + case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; - if( f=='b' ){ - int nInt = ((p->nCol + 31) / 32) * p->nPhrase; - for(i=0; inCol*p->nPhrase); i++) aOut[i*nMul] = 0; - } + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; for(iPhrase=0; iPhrasenPhrase; iPhrase++){ Fts5PhraseIter iter; diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 860cfedb9b..2ed10b7cdb 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -379,7 +379,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ - if( pCsr->aCnt[pCsr->iCol] ) break; + if( pCsr->aDoc[pCsr->iCol] ) break; } } @@ -412,24 +412,52 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ - rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); - if( rc==SQLITE_OK ){ - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; - } - pCsr->aDoc[0]++; - }else{ - int iCol = -1; - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; - if( iCol!=ii ){ - pCsr->aDoc[ii]++; - iCol = ii; + switch( pCsr->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); + if( rc==SQLITE_OK ){ + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + pCsr->aDoc[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + pCsr->aDoc[ii]++; + iCol = ii; + } + } } } - } + break; + + case FTS5_DETAIL_COLUMNS: + if( pTab->eType==FTS5_VOCAB_ROW ){ + pCsr->aDoc[0]++; + }else{ + Fts5Buffer buf = {0, 0, 0}; + rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); + if( rc==SQLITE_OK ){ + while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPosaDoc[iPos]++; + } + } + sqlite3Fts5BufferFree(&buf); + } + break; + + default: + assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + break; + } + + if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } @@ -445,7 +473,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ - while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++; + while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; assert( pCsr->iColpConfig->nCol ); } return rc; @@ -525,30 +553,36 @@ static int fts5VocabColumnMethod( int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; if( iCol==0 ){ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); - } - else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){ + }else if( eType==FTS5_VOCAB_COL ){ assert( iCol==1 || iCol==2 || iCol==3 ); if( iCol==1 ){ - const char *z = pCsr->pConfig->azCol[pCsr->iCol]; - sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } }else if( iCol==2 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]); + iVal = pCsr->aDoc[pCsr->iCol]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]); + iVal = pCsr->aCnt[pCsr->iCol]; } }else{ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[0]); + iVal = pCsr->aDoc[0]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[0]); + iVal = pCsr->aCnt[0]; } } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 2c64b3b9a4..2c7fedcefe 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -28,6 +28,28 @@ proc fts5_test_poslist {cmd} { set res } +proc fts5_test_poslist2 {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseForeach $i c o { + lappend res $i.$c.$o + } + } + + set res +} + +proc fts5_test_collist {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseColumnForeach $i c { lappend res $i.$c } + } + + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -113,6 +135,8 @@ proc fts5_aux_test_functions {db} { fts5_test_columntext fts5_test_columntotalsize fts5_test_poslist + fts5_test_poslist2 + fts5_test_collist fts5_test_tokenize fts5_test_rowcount fts5_test_all @@ -178,15 +202,24 @@ proc fts5_rnddoc {n} { # -near N (NEAR distance. Default 10) # -col C (List of column indexes to match against) # -pc VARNAME (variable in caller frame to use for phrase numbering) +# -dict VARNAME (array in caller frame to use for synonyms) # proc nearset {aCol args} { + + # Process the command line options. + # set O(-near) 10 set O(-col) {} set O(-pc) "" + set O(-dict) "" set nOpt [lsearch -exact $args --] if {$nOpt<0} { error "no -- option" } + # Set $lPhrase to be a list of phrases. $nPhrase its length. + set lPhrase [lrange $args [expr $nOpt+1] end] + set nPhrase [llength $lPhrase] + foreach {k v} [lrange $args 0 [expr $nOpt-1]] { if {[info exists O($k)]==0} { error "unrecognized option $k" } set O($k) $v @@ -198,9 +231,7 @@ proc nearset {aCol args} { upvar $O(-pc) counter } - # Set $phraselist to be a list of phrases. $nPhrase its length. - set phraselist [lrange $args [expr $nOpt+1] end] - set nPhrase [llength $phraselist] + if {$O(-dict)!=""} { upvar $O(-dict) aDict } for {set j 0} {$j < [llength $aCol]} {incr j} { for {set i 0} {$i < $nPhrase} {incr i} { @@ -208,27 +239,54 @@ proc nearset {aCol args} { } } - set iCol -1 - foreach col $aCol { - incr iCol - if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue - set nToken [llength $col] + # Loop through each column of the current row. + for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { - set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] - for { } {$iFL < $nToken} {incr iFL} { - for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { - set B($iPhrase) [list] - } + # If there is a column filter, test whether this column is excluded. If + # so, skip to the next iteration of this loop. Otherwise, set zCol to the + # column value and nToken to the number of tokens that comprise it. + if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue + set zCol [lindex $aCol $iCol] + set nToken [llength $zCol] + + # Each iteration of the following loop searches a substring of the + # column value for phrase matches. The last token of the substring + # is token $iLast of the column value. The first token is: + # + # iFirst = ($iLast - $O(-near) - 1) + # + # where $sz is the length of the phrase being searched for. A phrase + # counts as matching the substring if its first token lies on or before + # $iLast and its last token on or after $iFirst. + # + # For example, if the query is "NEAR(a+b c, 2)" and the column value: + # + # "x x x x A B x x C x" + # 0 1 2 3 4 5 6 7 8 9" + # + # when (iLast==8 && iFirst=5) the range will contain both phrases and + # so both instances can be added to the output poslists. + # + set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] + for { } {$iLast < $nToken} {incr iLast} { + + catch { array unset B } for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { - set p [lindex $phraselist $iPhrase] + set p [lindex $lPhrase $iPhrase] set nPm1 [expr {[llength $p] - 1}] - set iFirst [expr $iFL - $O(-near) - [llength $p]] + set iFirst [expr $iLast - $O(-near) - [llength $p]] - for {set i $iFirst} {$i <= $iFL} {incr i} { - if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i } + for {set i $iFirst} {$i <= $iLast} {incr i} { + set lCand [lrange $zCol $i [expr $i+$nPm1]] + set bMatch 1 + foreach tok $p term $lCand { + if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break } + } + if {$bMatch} { lappend B($iPhrase) $i } } - if {[llength $B($iPhrase)] == 0} break + + if {![info exists B($iPhrase)]} break } if {$iPhrase==$nPhrase} { @@ -252,10 +310,22 @@ proc nearset {aCol args} { incr counter } - #puts $res + #puts "$aCol -> $res" sort_poslist $res } +proc nearset_match {aDictVar tok term} { + if {[string match $tok $term]} { return 1 } + + upvar $aDictVar aDict + if {[info exists aDict($tok)]} { + foreach s $aDict($tok) { + if {[string match $s $term]} { return 1 } + } + } + return 0; +} + #------------------------------------------------------------------------- # Usage: # @@ -327,3 +397,167 @@ proc fts5_tokenize_split {text} { set ret } +#------------------------------------------------------------------------- +# +proc foreach_detail_mode {prefix script} { + set saved $::testprefix + foreach d [list full col none] { + set s [string map [list %DETAIL% $d] $script] + set ::detail $d + set ::testprefix "$prefix-$d" + reset_db + uplevel $s + unset ::detail + } + set ::testprefix $saved +} + +proc detail_check {} { + if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { + error "not in foreach_detail_mode {...} block" + } +} +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } +proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } + + +#------------------------------------------------------------------------- +# Convert a poslist of the type returned by fts5_test_poslist() to a +# collist as returned by fts5_test_collist(). +# +proc fts5_poslist2collist {poslist} { + set res [list] + foreach h $poslist { + regexp {(.*)\.[1234567890]+} $h -> cand + lappend res $cand + } + set res [lsort -command fts5_collist_elem_compare -unique $res] + return $res +} + +# Comparison function used by fts5_poslist2collist to sort collist entries. +proc fts5_collist_elem_compare {a b} { + foreach {a1 a2} [split $a .] {} + foreach {b1 b2} [split $b .] {} + + if {$a1==$b1} { return [expr $a2 - $b2] } + return [expr $a1 - $b1] +} + + +#-------------------------------------------------------------------------- +# Construct and return a tcl list equivalent to that returned by the SQL +# query executed against database handle [db]: +# +# SELECT +# rowid, +# fts5_test_poslist($tbl), +# fts5_test_collist($tbl) +# FROM $tbl('$expr') +# ORDER BY rowid $order; +# +proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} { + + # Figure out the set of columns in the FTS5 table. This routine does + # not handle tables with UNINDEXED columns, but if it did, it would + # have to be here. + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set d "" + if {$aDictVar != ""} { + upvar $aDictVar aDict + set d aDict + } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] ) + }]] + + set res [list] + db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { + set cols [list] + foreach col $lCols { lappend cols $x($col) } + + set ::pc 0 + set rowdata [eval $tclexpr] + if {$rowdata != ""} { + lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata] + } + } + + set res +} + +#------------------------------------------------------------------------- +# Similar to [fts5_query_data], but omit the collist field. +# +proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} { + set res [list] + + if {$aDictVar!=""} { + upvar $aDictVar aDict + set dict aDict + } else { + set dict "" + } + + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { + lappend res $rowid $poslist + } + set res +} + +#------------------------------------------------------------------------- +# + +# This command will only work inside a [foreach_detail_mode] block. It tests +# whether or not expression $expr run on FTS5 table $tbl is supported by +# the current mode. If so, 1 is returned. If not, 0. +# +# detail=full (all queries supported) +# detail=col (all but phrase queries and NEAR queries) +# detail=none (all but phrase queries, NEAR queries, and column filters) +# +proc fts5_expr_ok {expr tbl} { + + if {![detail_is_full]} { + set nearset "nearset_rc" + if {[detail_is_col]} { set nearset "nearset_rf" } + + set ::expr_not_ok 0 + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set ::pc 0 + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] ) + }]] + eval $tclexpr + if {$::expr_not_ok} { return 0 } + } + + return 1 +} + +# Helper for [fts5_expr_ok] +proc nearset_rf {aCol args} { + set idx [lsearch -exact $args --] + if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { + set ::expr_not_ok 1 + } + list +} + +# Helper for [fts5_expr_ok] +proc nearset_rc {aCol args} { + nearset_rf $aCol {*}$args + if {[lsearch $args -col]>=0} { + set ::expr_not_ok 1 + } + list +} + diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 59b8ab86d1..fdcf08398d 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -21,6 +21,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; @@ -41,9 +43,9 @@ do_execsql_test 1.1 { #------------------------------------------------------------------------- # -reset_db + do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); @@ -66,11 +68,12 @@ do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } + #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } foreach {i x y} { 1 {g f d b f} {h h e i a} @@ -93,7 +96,7 @@ foreach {i x y} { # reset_db do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -117,7 +120,7 @@ foreach {i x y} { # reset_db do_execsql_test 5.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -141,7 +144,7 @@ foreach {i x y} { # reset_db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -276,7 +279,7 @@ for {set i 1} {$i <= 10} {incr i} { # reset_db do_execsql_test 10.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } set d10 { 1 {g f d b f} {h h e i a} @@ -309,19 +312,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { - CREATE VIRTUAL TABLE rank USING fts5(a, b, c); + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { - CREATE VIRTUAL TABLE t2 USING fts5(x,y); + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); } {} do_catchsql_test 12.2 { @@ -337,7 +340,7 @@ do_test 12.3 { # reset_db do_execsql_test 13.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} @@ -361,7 +364,7 @@ do_execsql_test 13.6 { # reset_db do_execsql_test 14.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' @@ -371,6 +374,10 @@ do_execsql_test 14.1 { INSERT INTO t1 SELECT * FROM d LIMIT 200; } +do_execsql_test 15.x { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + do_test 14.2 { set nRow 0 db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { @@ -417,11 +424,11 @@ do_execsql_test 16.1 { } proc funk {} { + db eval { UPDATE n1_config SET v=50 WHERE k='version' } set fd [db incrblob main n1_data block 10] fconfigure $fd -encoding binary -translation binary puts -nonewline $fd "\x44\x45" close $fd - db eval { UPDATE n1_config SET v=50 WHERE k='version' } } db func funk funk @@ -433,7 +440,7 @@ do_catchsql_test 16.2 { # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE b2 USING fts5(x); + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); @@ -447,18 +454,20 @@ do_test 17.2 { set res } {{a b c} {a b c} {a b c}} -reset_db -do_execsql_test 18.1 { - CREATE VIRTUAL TABLE c2 USING fts5(x, y); - INSERT INTO c2 VALUES('x x x', 'x x x'); - SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; -} {1} +if {[string match n* %DETAIL%]==0} { + reset_db + do_execsql_test 17.3 { + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO c2 VALUES('x x x', 'x x x'); + SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; + } {1} +} #------------------------------------------------------------------------- # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE uio USING fts5(ttt); + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; @@ -505,8 +514,8 @@ do_execsql_test 17.9 { #-------------------------------------------------------------------- # do_execsql_test 18.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); - CREATE VIRTUAL TABLE t2 USING fts5(c, d); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } @@ -522,7 +531,7 @@ do_execsql_test 18.3 { # reset_db do_execsql_test 19.0 { - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; @@ -533,7 +542,7 @@ do_execsql_test 19.0 { # reset_db do_execsql_test 20.0 { - CREATE VIRTUAL TABLE temp.tmp USING fts5(x); + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] @@ -545,6 +554,7 @@ do_test 20.1 { execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } } $::ids +} finish_test diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 0746e64326..95da2cd2eb 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('hello', 'world'); INSERT INTO t1 VALUES('one two', 'three four'); INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); @@ -57,7 +59,7 @@ do_execsql_test 1.6 { reset_db do_execsql_test 2.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); INSERT INTO t1 VALUES('one'); INSERT INTO t1 VALUES('two'); @@ -159,7 +161,7 @@ foreach {tn expr res} { # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE s1 USING fts5(x); + CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); } foreach {tn doc} [list \ 1 [string repeat {a x } 1500000] \ @@ -172,8 +174,14 @@ do_execsql_test 4.3 { SELECT rowid FROM s1 WHERE s1 MATCH 'x' } {1 2} -do_execsql_test 4.4 { - SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' +if {[detail_is_full]} { + do_execsql_test 4.4 { + SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' + } {1 2} +} + +do_execsql_test 4.5 { + SELECT rowid FROM s1 WHERE s1 MATCH 'a x' } {1 2} #------------------------------------------------------------------------- @@ -182,7 +190,7 @@ do_execsql_test 4.4 { # (L-2) is larger than it. # do_execsql_test 5.0 { - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -222,7 +230,7 @@ do_test 5.2 { do_test 5.3 { execsql { DROP TABLE s2; - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -241,7 +249,7 @@ do_test 5.4 { #------------------------------------------------------------------------- # do_execsql_test 6.0 { - CREATE VIRTUAL TABLE s3 USING fts5(x); + CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); BEGIN; INSERT INTO s3 VALUES('a b c'); INSERT INTO s3 VALUES('A B C'); @@ -276,13 +284,13 @@ do_test 6.4 { #------------------------------------------------------------------------- # set doc [string repeat "a b c " 500] -breakpoint do_execsql_test 7.0 { - CREATE VIRTUAL TABLE x1 USING fts5(x); + CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); INSERT INTO x1 VALUES($doc); } +} ;# foreach_detail_mode... finish_test diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 0de4848145..00b1328867 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} @@ -125,80 +127,21 @@ set data { 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} } -# Argument $expr is an FTS5 match expression designed to be executed against -# an FTS5 table with the following schema: -# -# CREATE VIRTUAL TABLE xy USING fts5(x, y); -# -# Assuming the table contains the same records as stored int the global -# $::data array (see above), this function returns a list containing one -# element for each match in the dataset. The elements are themselves lists -# formatted as follows: -# -# { ...} -# -# where each element is a list of phrase matches in the -# same form as returned by auxiliary scalar function fts5_test(). -# -proc matchdata {bPos expr {bAsc 1}} { - - set tclexpr [db one { - SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y') - }] - set res [list] - - #puts $tclexpr - foreach {id x y} $::data { - set cols [list $x $y] - set ::pc 0 - #set hits [lsort -command instcompare [eval $tclexpr]] - set hits [eval $tclexpr] - if {[llength $hits]>0} { - if {$bPos} { - lappend res [list $id $hits] - } else { - lappend res $id - } - } - } - - if {$bAsc} { - set res [lsort -integer -increasing -index 0 $res] - } else { - set res [lsort -integer -decreasing -index 0 $res] - } - - return [concat {*}$res] -} - -# -# End of test code -#------------------------------------------------------------------------- - -proc fts5_test_poslist {cmd} { - set res [list] - for {set i 0} {$i < [$cmd xInstCount]} {incr i} { - lappend res [string map {{ } .} [$cmd xInst $i]] - } - set res -} - - foreach {tn2 sql} { 1 {} 2 {BEGIN} } { reset_db - sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist + fts5_aux_test_functions db - do_execsql_test 1.0 { - CREATE VIRTUAL TABLE xx USING fts5(x,y); + do_execsql_test 1.$tn2.0 { + CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } execsql $sql - do_test $tn2.1.1 { + do_test 1.$tn2.1.1 { foreach {id x y} $data { execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } } @@ -207,153 +150,131 @@ foreach {tn2 sql} { #------------------------------------------------------------------------- - # Test phrase queries. # - foreach {tn phrase} { - 1 "o" - 2 "b q" - 3 "e a e" - 4 "m d g q q b k b w f q q p p" - 5 "l o o l v v k" - 6 "a" - 7 "b" - 8 "c" - 9 "no" - 10 "L O O L V V K" - } { - set expr "\"$phrase\"" - set res [matchdata 1 $expr] - - do_execsql_test $tn2.1.2.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - } - - #------------------------------------------------------------------------- - # Test some AND and OR queries. - # - foreach {tn expr} { - 1.1 "a AND b" - 1.2 "a+b AND c" - 1.3 "d+c AND u" - 1.4 "d+c AND u+d" - - 2.1 "a OR b" - 2.2 "a+b OR c" - 2.3 "d+c OR u" - 2.4 "d+c OR u+d" - - 3.1 { a AND b AND c } - } { - set res [matchdata 1 $expr] - do_execsql_test $tn2.2.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - } - - #------------------------------------------------------------------------- - # Queries on a specific column. - # - foreach {tn expr} { - 1.1 "x:a" - 1.2 "y:a" - 1.3 "x:b" - 1.4 "y:b" - 2.1 "{x}:a" - 2.2 "{y}:a" - 2.3 "{x}:b" - 2.4 "{y}:b" - - 3.1 "{x y}:a" - 3.2 "{y x}:a" - 3.3 "{x x}:b" - 3.4 "{y y}:b" - - 4.1 {{"x" "y"}:a} - 4.2 {{"y" x}:a} - 4.3 {{x "x"}:b} - 4.4 {{"y" y}:b} - } { - set res [matchdata 1 $expr] - do_execsql_test $tn2.3.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - } - - #------------------------------------------------------------------------- - # Some NEAR queries. - # - foreach {tn expr} { - 1 "NEAR(a b)" - 2 "NEAR(r c)" - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } - } { - set res [matchdata 1 $expr] - do_execsql_test $tn2.4.1.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - } - - do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0} - do_test $tn2.4.2 { nearset {{a b c}} -- c } {0.0.2} - - foreach {tn expr tclexpr} { - 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} - } { - do_execsql_test $tn2.5.$tn { - SELECT fts5_expr_tcl($expr, 'N $x') - } [list $tclexpr] - } - - #------------------------------------------------------------------------- - # - do_execsql_test $tn2.6.integrity { + do_execsql_test 1.$tn2.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } - #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} - foreach {bAsc sql} { - 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} - 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} + + #------------------------------------------------------------------------- + # + foreach {tn expr} { + 1.2 "a OR b" + 1.1 "a AND b" + 1.3 "o" + 1.4 "b q" + 1.5 "e a e" + 1.6 "m d g q q b k b w f q q p p" + 1.7 "l o o l v v k" + 1.8 "a" + 1.9 "b" + 1.10 "c" + 1.11 "no" + 1.12 "L O O L V V K" + 1.13 "a AND b AND c" + 1.14 "x:a" + + 2.1 "x:a" + 2.2 "y:a" + 2.3 "x:b" + 2.4 "y:b" + + 3.1 "{x}:a" + 3.2 "{y}:a" + 3.3 "{x}:b" + 3.4 "{y}:b" + + 4.1 "{x y}:a" + 4.2 "{y x}:a" + 4.3 "{x x}:b" + 4.4 "{y y}:b" + + 5.1 {{"x" "y"}:a} + 5.2 {{"y" x}:a} + 5.3 {{x "x"}:b} + 5.4 {{"y" y}:b} + + 6.1 "b + q" + 6.2 "e + a + e" + 6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" + 6.4 "l + o + o + l + v + v + k" + 6.5 "L + O + O + L + V + V + K" + + 7.1 "a+b AND c" + 7.2 "d+c AND u" + 7.3 "d+c AND u+d" + 7.4 "a+b OR c" + 7.5 "d+c OR u" + 7.6 "d+c OR u+d" + + 8.1 "NEAR(a b)" + 8.2 "NEAR(r c)" + 8.2 { NEAR(r c, 5) } + 8.3 { NEAR(r c, 3) } + 8.4 { NEAR(r c, 2) } + 8.5 { NEAR(r c, 0) } + 8.6 { NEAR(a b c) } + 8.7 { NEAR(a b c, 8) } + 8.8 { x : NEAR(r c) } + 8.9 { y : NEAR(r c) } + + 9.1 { NEAR(r c) } + 9.2 { NEAR(r c, 5) } + 9.3 { NEAR(r c, 3) } + 9.4 { NEAR(r c, 2) } + 9.5 { NEAR(r c, 0) } + 9.6 { NEAR(a b c) } + 9.7 { NEAR(a b c, 8) } + 9.8 { x : NEAR(r c) } + 9.9 { y : NEAR(r c) } + 9.10 { x : "r c" } + 9.11 { y : "r c" } + 9.12 { a AND b } + 9.13 { a AND b AND c } + 9.14a { a } + 9.14b { a OR b } + 9.15 { a OR b AND c } + 9.16 { c AND b OR a } + 9.17 { c AND (b OR a) } + 9.18 { c NOT (b OR a) } + 9.19 { (c NOT b) OR (a AND d) } } { - foreach {tn expr} { - 0.1 x - 1 { NEAR(r c) } - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } - 10 { x : "r c" } - 11 { y : "r c" } - 12 { a AND b } - 13 { a AND b AND c } - 14a { a } - 14b { a OR b } - 15 { a OR b AND c } - 16 { c AND b OR a } - 17 { c AND (b OR a) } - 18 { c NOT (b OR a) } - 19 { c NOT b OR a AND d } - } { - set res [matchdata 0 $expr $bAsc] - do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res + + if {[fts5_expr_ok $expr xx]==0} { + do_test 1.$tn2.$tn.OMITTED { list } [list] + continue } + + set res [fts5_query_data $expr xx] + do_execsql_test 1.$tn2.$tn.[llength $res].asc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr + } $res + + + set res [fts5_query_data $expr xx DESC] + do_execsql_test 1.$tn2.$tn.[llength $res].desc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr ORDER BY 1 DESC + } $res } } -do_execsql_test 3.1 { +} + +do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} +do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0} +do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2} + +foreach {tn expr tclexpr} { + 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} +} { + do_execsql_test 2.3.$tn { + SELECT fts5_expr_tcl($expr, 'N $x') + } [list $tclexpr] +} + finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index 3881c7e161..c6b09d8121 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE yy USING fts5(x, y); + CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); @@ -53,23 +55,23 @@ foreach {tn match res} { foreach {T create} { 2 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 3 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 4 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } 5 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } @@ -235,5 +237,7 @@ foreach {T create} { catchsql COMMIT } +} + finish_test diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index ded73d472f..5153306d19 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -55,7 +57,7 @@ fts5_aux_test_functions db #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(x, y); + CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); } @@ -76,31 +78,35 @@ do_execsql_test 2.2 { 2 {1.0.2 1.0.10} } -do_execsql_test 2.3 { - SELECT rowid, fts5_test_poslist(t2) FROM t2 - WHERE t2 MATCH 'y:o' ORDER BY rowid; -} { - 1 {0.1.3 0.1.7} +if {[detail_is_full]} { + do_execsql_test 2.3 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'y:o' ORDER BY rowid; + } { + 1 {0.1.3 0.1.7} + } } #------------------------------------------------------------------------- # do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t3 USING fts5(x, y); + CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); INSERT INTO t3 VALUES( 'r c', ''); } -do_execsql_test 3.1 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; -} { - 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} -} +if {[detail_is_full]} { + do_execsql_test 3.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; + } { + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} + } -do_execsql_test 3.2 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; -} { - 2 {0.0.0 1.0.1} + do_execsql_test 3.2 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; + } { + 2 {0.0.0 1.0.1} + } } do_execsql_test 3.3 { @@ -116,7 +122,7 @@ do_execsql_test 3.3 { #------------------------------------------------------------------------- # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t4 USING fts5(x, y); + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); } @@ -134,7 +140,7 @@ reset_db fts5_aux_test_functions db do_execsql_test 5.1 { - CREATE VIRTUAL TABLE t5 USING fts5(x, y); + CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); INSERT INTO t5 VALUES('', 'a'); INSERT INTO t5 VALUES('a', ''); @@ -182,7 +188,7 @@ do_execsql_test 5.5 { reset_db fts5_aux_test_functions db do_execsql_test 6.1 { - CREATE VIRTUAL TABLE t6 USING fts5(x, y); + CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } @@ -200,7 +206,7 @@ do_execsql_test 6.2 { reset_db fts5_aux_test_functions db do_execsql_test 7.1 { - CREATE VIRTUAL TABLE t7 USING fts5(x, y); + CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); } do_test 7.2 { foreach {x y} { @@ -240,7 +246,7 @@ do_execsql_test 7.4 { #------------------------------------------------------------------------- # do_test 8.1 { - execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) } + execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } foreach {rowid x y} { 0 {A o} {o o o C o o o o o o o o} 1 {o o B} {o o o C C o o o o o o o} @@ -300,5 +306,7 @@ foreach {tn q cnt} { } $cnt } +} + finish_test diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index 8c50f84866..d6b2241568 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -24,9 +24,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } proc do_snippet_test {tn doc match res} { @@ -111,34 +112,37 @@ foreach {tn doc res} { do_snippet_test 1.$tn $doc X $res } -foreach {tn doc res} { - 1.1 {X Y o o o o o} {[X Y] o o o o o} - 1.2 {o X Y o o o o} {o [X Y] o o o o} - 1.3 {o o X Y o o o} {o o [X Y] o o o} - 1.4 {o o o X Y o o} {o o o [X Y] o o} - 1.5 {o o o o X Y o} {o o o o [X Y] o} - 1.6 {o o o o o X Y} {o o o o o [X Y]} +if {[detail_is_full]} { + foreach {tn doc res} { + 1.1 {X Y o o o o o} {[X Y] o o o o o} + 1.2 {o X Y o o o o} {o [X Y] o o o o} + 1.3 {o o X Y o o o} {o o [X Y] o o o} + 1.4 {o o o X Y o o} {o o o [X Y] o o} + 1.5 {o o o o X Y o} {o o o o [X Y] o} + 1.6 {o o o o o X Y} {o o o o o [X Y]} - 2.1 {X Y o o o o o o} {[X Y] o o o o o...} - 2.2 {o X Y o o o o o} {o [X Y] o o o o...} - 2.3 {o o X Y o o o o} {o o [X Y] o o o...} - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} + 2.1 {X Y o o o o o o} {[X Y] o o o o o...} + 2.2 {o X Y o o o o o} {o [X Y] o o o o...} + 2.3 {o o X Y o o o o} {o o [X Y] o o o...} + 2.4 {o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o X Y} {...o o o o o [X Y]} - 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} - 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} - 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} - -} { - do_snippet_test 2.$tn $doc "X + Y" $res + 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} + 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} + 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} + 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} + } { + do_snippet_test 2.$tn $doc "X + Y" $res + } } +} ;# foreach_detail_mode + finish_test diff --git a/ext/fts5/test/fts5ag.test b/ext/fts5/test/fts5ag.test index 42a588f56c..de126a25f0 100644 --- a/ext/fts5/test/fts5ag.test +++ b/ext/fts5/test/fts5ag.test @@ -33,8 +33,10 @@ ifcapable !fts5 { # ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] # +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, z); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%); } do_test 1.1 { @@ -119,19 +121,24 @@ foreach {tn expr} { 2.3 c 2.4 d - 2.5 {"m m"} - 2.6 {e + s} - 3.0 {a AND b} 3.1 {a OR b} 3.2 {b OR c AND d} - 3.3 {NEAR(c d)} } { do_fts5ag_test $tn $expr - - if {[set_test_counter errors]} break } +if {[detail_is_full]} { + foreach {tn expr} { + 4.1 {"m m"} + 4.2 {e + s} + 4.3 {NEAR(c d)} + } { + do_fts5ag_test $tn $expr + } +} + +} ;# foreach_detail_mode finish_test diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 6d7e39f793..b7beb5655b 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -21,12 +21,16 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + #------------------------------------------------------------------------- # This file contains tests for very large doclists. # +set Y [list] +set W [list] do_test 1.0 { - execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) } + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) } execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } set v {w w w w w w w w w w w w w w w w w w w w} execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) } @@ -70,7 +74,12 @@ do_test 1.4 { set nRead [reads] execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } set nReadX [expr [reads] - $nRead] - expr $nReadX>1000 + #puts -nonewline "(nReadX=$nReadX)" + if {[detail_is_full]} { set expect 1000 } + if {[detail_is_col]} { set expect 250 } + if {[detail_is_none]} { set expect 80 } + + expr $nReadX>$expect } {1} do_test 1.5 { @@ -87,17 +96,22 @@ foreach {tn q res} " 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] " { + if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue + + if {[detail_is_full]} { set ratio 8 } + if {[detail_is_col]} { set ratio 4 } + if {[detail_is_none]} { set ratio 2 } do_test 1.6.$tn.1 { set n [execsql_reads $q] #puts -nonewline "(n=$n nReadX=$nReadX)" - expr {$n < ($nReadX / 8)} + expr {$n < ($nReadX / $ratio)} } {1} do_test 1.6.$tn.2 { set n [execsql_reads "$q ORDER BY rowid DESC"] #puts -nonewline "(n=$n nReadX=$nReadX)" - expr {$n < ($nReadX / 8)} + expr {$n < ($nReadX / $ratio)} } {1} do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res] @@ -109,21 +123,26 @@ foreach {tn q res} " # number of pages loaded from disk. # foreach {tn fraction tail cnt} { - 1 0.6 {rowid > 5000} 5000 - 2 0.2 {rowid > 9000} 1000 - 3 0.2 {rowid < 1000} 999 - 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 - 5 0.6 {rowid >= 5000} 5001 - 6 0.2 {rowid >= 9000} 1001 - 7 0.2 {rowid <= 1000} 1000 - 8 0.6 {rowid > '5000'} 5000 - 9 0.2 {rowid > '9000'} 1000 + 1 0.6 {rowid > 5000} 5000 + 2 0.2 {rowid > 9000} 1000 + 3 0.2 {rowid < 1000} 999 + 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 + 5 0.6 {rowid >= 5000} 5001 + 6 0.2 {rowid >= 9000} 1001 + 7 0.2 {rowid <= 1000} 1000 + 8 0.6 {rowid > '5000'} 5000 + 9 0.2 {rowid > '9000'} 1000 10 0.1 {rowid = 444} 1 } { set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" set n [execsql_reads $q] set ret [llength [execsql $q]] + # Because the position lists for 'x' are quite long in this db, the + # advantage is a bit smaller in detail=none mode. Update $fraction to + # reflect this. + if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] } + do_test "1.7.$tn.asc.(n=$n ret=$ret)" { expr {$n < ($fraction*$nReadX) && $ret==$cnt} } {1} @@ -143,6 +162,7 @@ do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} +} ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5ai.test b/ext/fts5/test/fts5ai.test index 63c46fd042..e32c806c46 100644 --- a/ext/fts5/test/fts5ai.test +++ b/ext/fts5/test/fts5ai.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a); + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%); } {} do_execsql_test 1.1 { @@ -49,6 +51,7 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } +} finish_test diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index 4eb28324c9..0f699a601f 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.1 { - CREATE VIRTUAL TABLE ft1 USING fts5(x); + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); INSERT INTO ft1 VALUES('i d d a g i b g d d'); INSERT INTO ft1 VALUES('h d b j c c g a c a'); INSERT INTO ft1 VALUES('e j a e f h b f h h'); @@ -35,6 +37,9 @@ do_execsql_test 1.1 { INSERT INTO ft1 VALUES('i c c f a d g h j e'); INSERT INTO ft1 VALUES('i d i g c d c h b f'); INSERT INTO ft1 VALUES('g d a e h a b c f j'); + + CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%); + INSERT INTO ft2 VALUES('a b c d e f g h i j'); } do_execsql_test 1.2 { @@ -49,19 +54,6 @@ do_execsql_test 1.2 { } do_execsql_test 1.3 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; -} { - {[h d] b j c c g a c a} - {j f [h d] g h i b d f} -} - -do_execsql_test 1.4 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; -} { - {i [d d] a g i b g [d d]} -} - -do_execsql_test 1.5 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' } { {[e] j a [e] f h b f h h} @@ -72,57 +64,71 @@ do_execsql_test 1.5 { {g d a [e] h a b c f j} } -do_execsql_test 1.6 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; -} { - {i [d d] a g i b g [d d]} -} - -do_execsql_test 2.1 { - CREATE VIRTUAL TABLE ft2 USING fts5(x); - INSERT INTO ft2 VALUES('a b c d e f g h i j'); -} - -do_execsql_test 2.2 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' -} {{a [b c d e] f g h i j}} - -do_execsql_test 2.3 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' -} { - {a [b c d] [e f g] h i j} -} - -do_execsql_test 2.4 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' -} { - {a [b c d] e f g h i j} -} - -do_execsql_test 2.5 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' -} { - {a [b c d e] f g h i j} -} - -do_execsql_test 2.6.1 { +do_execsql_test 1.4 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' } { {a b c [d] e [f] g h i j} } -do_execsql_test 2.6.2 { +do_execsql_test 1.5 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' } { {a b c [d] e [f] g h i j} } +#------------------------------------------------------------------------- +# Tests below this point require detail=full. +#------------------------------------------------------------------------- +if {[detail_is_full]==0} continue + + +do_execsql_test 2.1 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; +} { + {[h d] b j c c g a c a} + {j f [h d] g h i b d f} +} + +do_execsql_test 2.2 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.3 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.4 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' +} {{a [b c d e] f g h i j}} + +do_execsql_test 2.5 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' +} { + {a [b c d] [e f g] h i j} +} + +do_execsql_test 2.6 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' +} { + {a [b c d] e f g h i j} +} + +do_execsql_test 2.7 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' +} { + {a [b c d e] f g h i j} +} + #------------------------------------------------------------------------- # The example from the docs. # do_execsql_test 3.1 { -- Assuming this: - CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%); INSERT INTO ft VALUES('a b c x c d e'); INSERT INTO ft VALUES('a b c c d e'); INSERT INTO ft VALUES('a b c d e'); @@ -138,6 +144,7 @@ do_execsql_test 3.1 { {[a b c d e]} } +} finish_test diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index b8f8c6ebcd..c0dd2117dd 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.1 { - CREATE VIRTUAL TABLE ft1 USING fts5(x); + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); SELECT * FROM ft1_config; } {version 4} @@ -83,7 +85,7 @@ foreach {tn defn} { # do_execsql_test 3.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('q w e r t y'); INSERT INTO t1 VALUES('y t r e w q'); } @@ -122,11 +124,13 @@ do_execsql_test 3.4.1 { {{0 0 5}} } -do_execsql_test 3.4.2 { - SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' -} { - {{1 0 1}} - {{0 0 2} {1 0 4}} +if {[detail_is_full]} { + do_execsql_test 3.4.2 { + SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' + } { + {{1 0 1}} + {{0 0 2} {1 0 4}} + } } proc coltest {cmd} { @@ -149,7 +153,7 @@ do_execsql_test 3.5.1 { # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e'); INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q'); INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t'); @@ -218,24 +222,26 @@ proc rowidplus {cmd ival} { } sqlite3_fts5_create_function db rowidplus rowidplus -do_execsql_test 4.2.1 { - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' -} { - 10 110 -} -do_execsql_test 4.2.2 { - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' -} { - 10 121 -} +if {[detail_is_full]} { + do_execsql_test 4.2.1 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 110 + } + do_execsql_test 4.2.2 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 121 + } -do_execsql_test 4.2.3 { - SELECT rowid, rank FROM t2 - WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' -} { - 10 122 + do_execsql_test 4.2.3 { + SELECT rowid, rank FROM t2 + WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' + } { + 10 122 + } } proc rowidmod {cmd imod} { @@ -243,7 +249,7 @@ proc rowidmod {cmd imod} { } sqlite3_fts5_create_function db rowidmod rowidmod do_execsql_test 4.3.1 { - CREATE VIRTUAL TABLE t3 USING fts5(x); + CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%); INSERT INTO t3 VALUES('a one'); INSERT INTO t3 VALUES('a two'); INSERT INTO t3 VALUES('a three'); @@ -287,6 +293,7 @@ do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} +} ;# foreach_detail_mode finish_test diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test index 771a0b64d8..218b3f4862 100644 --- a/ext/fts5/test/fts5auto.test +++ b/ext/fts5/test/fts5auto.test @@ -22,7 +22,6 @@ ifcapable !fts5 { return } - set data { -4026076 {n x w k b p x b n t t d s} {f j j s p j o} @@ -232,37 +231,9 @@ do_execsql_test 1.0 { fts5_aux_test_functions db -proc matchdata {expr tbl collist {order ASC}} { - - set cols "" - foreach e $collist { - append cols ", '$e'" - } - - set tclexpr [db one [subst -novar { - SELECT fts5_expr_tcl( - $expr, 'nearset $cols -pc ::pc' [set cols] - ) - }]] - set res [list] - - db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { - set cols [list] - foreach col $x(*) { - if {$col != "rowid"} { lappend cols $x($col) } - } - # set cols [list $a $b $c $d $e $f] - set ::pc 0 - set rowdata [eval $tclexpr] - if {$rowdata != ""} { lappend res $x(rowid) $rowdata } - } - - set res -} - -proc do_auto_test {tn tbl cols expr} { +proc do_auto_test {tn tbl expr} { foreach order {asc desc} { - set res [matchdata $expr $tbl $cols $order] + set res [fts5_poslist_data $expr $tbl $order] set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]" set ::autotest_expr $expr @@ -271,8 +242,6 @@ proc do_auto_test {tn tbl cols expr} { WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order] }] $res } - - } #------------------------------------------------------------------------- @@ -332,7 +301,7 @@ for {set fold 0} {$fold < 3} {incr fold} { C.1 { a OR (b AND "b c") } C.2 { a OR (b AND "z c") } } { - do_auto_test 3.$fold.$tn tt {a b c d e f} $expr + do_auto_test 3.$fold.$tn tt $expr } } @@ -366,11 +335,8 @@ foreach {tn expr} { 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} - - } { -breakpoint - do_auto_test 4.$tn yy {c1 c2 c3} $expr + do_auto_test 4.$tn yy $expr } diff --git a/ext/fts5/test/fts5detail.test b/ext/fts5/test/fts5detail.test new file mode 100644 index 0000000000..5bdd14424e --- /dev/null +++ b/ext/fts5/test/fts5detail.test @@ -0,0 +1,240 @@ +# 2015 December 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5detail + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +fts5_aux_test_functions db + +#-------------------------------------------------------------------------- +# Simple tests. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); + INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 + INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 + INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 + INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 + INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 + INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 + INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 + INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8 + INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9 + INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10 +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +foreach {tn match res} { + 1 "a:a" {9} + 2 "b:g" {1 4 6} + 3 "c:h" {1 3 6 8 9 10} +} { + do_execsql_test 1.2.$tn.1 { + SELECT rowid FROM t1($match); + } $res + + do_execsql_test 1.2.$tn.2 { + SELECT rowid FROM t1($match || '*'); + } $res +} + +do_catchsql_test 1.3.1 { + SELECT rowid FROM t1('h + d'); +} {1 {fts5: phrase queries are not supported (detail!=full)}} + +do_catchsql_test 1.3.2 { + SELECT rowid FROM t1('NEAR(h d)'); +} {1 {fts5: NEAR queries are not supported (detail!=full)}} + + +#------------------------------------------------------------------------- +# integrity-check with both detail= and prefix= options. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); + INSERT INTO t2(a) VALUES('aa ab'); +} + +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} + +do_execsql_test 2.1 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 2.2 { + SELECT fts5_test_poslist(t2) FROM t2('aa'); +} {0.0.0} + +set ::pc 0 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] +#exit + +#------------------------------------------------------------------------- +# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs +# work with detail=col tables. +# +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} + 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} + 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} + 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} + 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} + 6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb} + 7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc} + 8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb} + 9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb} + 10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa} + 11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac} + 12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc} + 13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc} + 14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa} + 15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc} + 16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc} + 17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca} + 18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc} + 19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc} + 20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc} + 21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb} + 22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc} + 23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc} + 24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb} + 25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc} + 26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa} + 27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac} + 28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa} + 29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac} + 30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac} + 31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba} + 32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc} + 33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa} + 34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac} + 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} + 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} + 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} + 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} + 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} + 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} +} + +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} +} + +proc matchdata {expr {bAsc 1}} { + + set tclexpr [db one { + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') + }] + set res [list] + + #puts "$expr -> $tclexpr" + foreach {id x y z} $::data { + set cols [list $x $y $z] + set ::pc 0 + #set hits [lsort -command instcompare [eval $tclexpr]] + set hits [eval $tclexpr] + if {[llength $hits]>0} { + lappend res [list $id $hits] + } + } + + if {$bAsc} { + set res [lsort -integer -increasing -index 0 $res] + } else { + set res [lsort -integer -decreasing -index 0 $res] + } + + return [concat {*}$res] +} + +foreach {tn tbl} { + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } + 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } +} { + reset_db + fts5_aux_test_functions db + execsql $tbl + foreach {id x y z} $data { + execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } + } + foreach {tn2 expr} { + 1 aaa 2 ccc 3 bab 4 aac + 5 aa* 6 cc* 7 ba* 8 aa* + 9 a* 10 b* 11 c* + } { + + set res [matchdata $expr] + + do_execsql_test 3.$tn.$tn2.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) + } $res + + do_execsql_test 3.$tn.$tn2.2 { + SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) + } $res + } +} + +#------------------------------------------------------------------------- +# Simple tests for detail=none tables. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); +} + +do_catchsql_test 4.1 { + SELECT * FROM t4('a:a') +} {1 {fts5: column queries are not supported (detail=none)}} + +#------------------------------------------------------------------------- +# Test that for the same content detail=none uses less space than +# detail=col, and that detail=col uses less space than detail=full +# +reset_db +do_test 5.1 { + foreach {tbl detail} {t1 none t2 col t3 full} { + execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" + foreach {rowid x y z} $::data { + execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" + } + } +} {} + +do_execsql_test 5.2 { + SELECT + (SELECT sum(length(block)) from t1_data) < + (SELECT sum(length(block)) from t2_data) +} {1} + +do_execsql_test 5.3 { + SELECT + (SELECT sum(length(block)) from t2_data) < + (SELECT sum(length(block)) from t3_data) +} {1} + + + +finish_test + diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 5a03c6989b..232b5021f1 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -26,7 +26,7 @@ if { $tcl_platform(wordSize)<8 } { return } -if 1 { +foreach_detail_mode $testprefix { proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] @@ -34,7 +34,7 @@ proc do_fb_test {tn sql res} { uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] } -# This test populates the FTS5 table containing $nEntry entries. Rows are +# This test populates the FTS5 table with $nEntry entries. Rows are # numbered from 0 to ($nEntry-1). The rowid for row $i is: # # ($iFirst + $i*$nStep) @@ -77,10 +77,12 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} { do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc - do_fb_test $tn.5.1 { - SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc - do_fb_test $tn.5.2 { - SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc + if {[detail_is_full]} { + do_fb_test $tn.5.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc + do_fb_test $tn.5.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc + } } @@ -90,7 +92,7 @@ foreach {tn pgsz} { } { do_execsql_test $tn.0 { DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); } @@ -107,7 +109,7 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { execsql { BEGIN; DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 64); INSERT INTO t1 VALUES('b a'); @@ -130,8 +132,6 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] -} - #-------------------------------------------------------------------- # reset_db @@ -158,7 +158,7 @@ proc rnddoc {} { db func rnddoc rnddoc do_execsql_test 3.1 { - CREATE VIRTUAL TABLE abc USING fts5(a); + CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%); INSERT INTO abc(abc, rank) VALUES('pgsz', 32); INSERT INTO abc VALUES ( rnddoc() ); @@ -192,6 +192,9 @@ foreach v $vocab { } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } +} ;# foreach_detail_mode + + finish_test diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 55c4b15cf3..8205396047 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -42,6 +42,15 @@ foreach {tn expr res} { 10 {abc + "" + def} {"abc" + "def"} 11 {abc "" def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} + + 13 {a AND b NOT c} {"a" AND ("b" NOT "c")} + 14 {a OR b NOT c} {"a" OR ("b" NOT "c")} + 15 {a NOT b AND c} {("a" NOT "b") AND "c"} + 16 {a NOT b OR c} {("a" NOT "b") OR "c"} + + 17 {a AND b OR c} {("a" AND "b") OR "c"} + 18 {a OR b AND c} {"a" OR ("b" AND "c")} + } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test new file mode 100644 index 0000000000..01a1876955 --- /dev/null +++ b/ext/fts5/test/fts5fault8.test @@ -0,0 +1,49 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault8 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +if {[detail_is_none]==0} continue + +fts5_aux_test_functions db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); + INSERT INTO t1 VALUES('a b a b', NULL); + INSERT INTO t1 VALUES(NULL, '1 2 1 2'); +} + +do_faultsim_test 1 -faults oom-t* -body { + execsql { + SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' + } +} -test { + faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ + {1 SQLITE_NOMEM} +} + +} + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 21f9b003e7..06f4550b47 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -16,6 +16,8 @@ set testprefix fts5matchinfo # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test ; return } +foreach_detail_mode $testprefix { + proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -27,7 +29,7 @@ db func mit mit sqlite3_fts5_register_matchinfo db do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(content); + CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%); } do_execsql_test 1.1 { @@ -41,7 +43,7 @@ do_execsql_test 1.1 { # Now create an FTS4 table that does not specify matchinfo=fts3. # do_execsql_test 1.2 { - CREATE VIRTUAL TABLE t2 USING fts5(content); + CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%); INSERT INTO t2 SELECT * FROM t1; SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; } {{1 1 1 2 2} {1 1 1 2 2}} @@ -149,9 +151,17 @@ proc normalize2 {list_of_lists} { return $res } +# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5 +# mode is not detail=full. +# +proc do_matchinfo_p_test {tn tbl expr results} { + if {[detail_is_full]} { + uplevel [list do_matchinfo_test $tn $tbl $expr $results] + } +} do_execsql_test 4.1.0 { - CREATE VIRTUAL TABLE t4 USING fts5(x, y); + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); } @@ -185,7 +195,7 @@ do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { xpxsscplax - } -do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { +do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} { p {1 1} c {2 2} x { @@ -203,8 +213,8 @@ do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { } do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } -do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } -do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { x { @@ -220,7 +230,7 @@ do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} { } do_execsql_test 4.2.0 { - CREATE VIRTUAL TABLE t5 USING fts5(content); + CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%); INSERT INTO t5 VALUES('a a a a a'); INSERT INTO t5 VALUES('a b a b a'); INSERT INTO t5 VALUES('c b c b c'); @@ -233,7 +243,7 @@ do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; @@ -250,7 +260,7 @@ if 0 { do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } @@ -260,10 +270,10 @@ do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_execsql_test 4.5.0 { - CREATE VIRTUAL TABLE t6 USING fts5(a, b, c); + CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%); INSERT INTO t6 VALUES('a', 'b', 'c'); } do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } @@ -274,7 +284,7 @@ do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } # use the full-text index (i.e. lookup by rowid or full-table scan). # do_execsql_test 7.1 { - CREATE VIRTUAL TABLE t10 USING fts5(content); + CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%); INSERT INTO t10 VALUES('first record'); INSERT INTO t10 VALUES('second record'); } @@ -299,7 +309,7 @@ do_execsql_test 7.4 { # SELECT sum(length(content)) < count(*) FROM fts4table; # do_execsql_test 8.1 { - CREATE VIRTUAL TABLE t11 USING fts5(content); + CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%); INSERT INTO t11(t11, rank) VALUES('pgsz', 32); INSERT INTO t11 VALUES('quitealongstringoftext'); INSERT INTO t11 VALUES('anotherquitealongstringoftext'); @@ -318,22 +328,24 @@ do_execsql_test 8.3 { #------------------------------------------------------------------------- -do_execsql_test 9.1 { - CREATE VIRTUAL TABLE t12 USING fts5(content); - INSERT INTO t12 VALUES('a b c d'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} {{0 1 1 0 1 1 1 1 1}} -do_execsql_test 9.2 { - INSERT INTO t12 VALUES('a d c d'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} { - {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} -} -do_execsql_test 9.3 { - INSERT INTO t12 VALUES('a d d a'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} { - {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} +if {[detail_is_full]} { + do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%); + INSERT INTO t12 VALUES('a b c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } {{0 1 1 0 1 1 1 1 1}} + do_execsql_test 9.2 { + INSERT INTO t12 VALUES('a d c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} + } + do_execsql_test 9.3 { + INSERT INTO t12 VALUES('a d d a'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} + } } #--------------------------------------------------------------------------- @@ -341,7 +353,7 @@ do_execsql_test 9.3 { # do_execsql_test 10.1 { DROP TABLE t10; - CREATE VIRTUAL TABLE t10 USING fts5(idx, value); + CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%); INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); SELECT t10.rowid, t10.* FROM t10 @@ -358,7 +370,7 @@ do_execsql_test 10.1 { reset_db sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { - CREATE VIRTUAL TABLE tt USING fts5(x, y); + CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 @@ -410,6 +422,8 @@ foreach {tn expr res} { } } { + + if {[string match *:* $expr] && [detail_is_none]} continue do_execsql_test 11.1.$tn.1 { SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr } $res @@ -443,7 +457,7 @@ db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } - execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)" } {} do_execsql_test 12.1 { @@ -451,5 +465,7 @@ do_execsql_test 12.1 { SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] +} ;# foreach_detail_mode + finish_test diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 27a1aee08b..e29c13117c 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -18,7 +18,9 @@ ifcapable !fts5 { finish_test return } - + + if 1 { + #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" @@ -350,6 +352,35 @@ do_execsql_test 4.1 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' } {0 {} 4} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 15.0 { + CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); + INSERT INTO x2 VALUES('ab'); +} + +do_execsql_test 15.1 { + INSERT INTO x2(x2) VALUES('integrity-check'); +} + +} + +#------------------------------------------------------------------------- +foreach_detail_mode $testprefix { + reset_db + fts5_aux_test_functions db + do_execsql_test 16.0 { + CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); + INSERT INTO x3 VALUES('a b c d e f'); + } + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); + } {2.0.2} + + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('a OR c'); + } {{0.0.0 1.0.2}} +} finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test new file mode 100644 index 0000000000..c61970e154 --- /dev/null +++ b/ext/fts5/test/fts5simple2.test @@ -0,0 +1,304 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple2 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if 1 { + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c'); +} +do_execsql_test 1.1 { + SELECT rowid FROM t1('c a b') +} {1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); + COMMIT; +} +do_execsql_test 2.1 { + SELECT rowid FROM t1('b c d') +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); +} +do_execsql_test 3.1 { + SELECT rowid FROM t1('b c d'); COMMIT; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a3 b3 c3'); + COMMIT; +} +do_execsql_test 4.1 { + SELECT rowid FROM t1('b*'); +} {1 2 3} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); + COMMIT; +} +do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a1 b1 c1'); + COMMIT; +} + +do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1'); + INSERT INTO t1 VALUES('a1 b2'); + COMMIT; +} +do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} +do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); +} +do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2} + +do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} +do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} + +do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') } + +do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2} + +do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} +do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} +do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 9.0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); +} +do_execsql_test 9.0.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +reset_db +do_execsql_test 9.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1', 'x y z'); + INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3'); + INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z'); +} +do_execsql_test 9.2.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('b1'); + INSERT INTO t1 VALUES('b1'); + DELETE FROM t1 WHERE rowid=1; +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1('b1'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t1('b1') ORDER BY rowid DESC; +} {2} + +do_execsql_test 10.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH d(x,y) AS ( + SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d + ) + INSERT INTO t1 SELECT * FROM d LIMIT 23; +} + +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } +do_execsql_test 11.2 { + SELECT rowid FROM t1; +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.3 { + SELECT rowid FROM t1('xyz'); +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); + INSERT INTO yy VALUES('in if'); + INSERT INTO yy VALUES('if'); +} {} + +do_execsql_test 12.1 { + SELECT rowid FROM yy('i*'); +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); +} {} +foreach {rowid a} { + 0 {f} + 1 {u} + 2 {k} + 3 {a} + 4 {a} + 5 {u} + 6 {u} + 7 {u} + 8 {f} + 9 {f} + 10 {a} + 11 {p} + 12 {f} + 13 {u} + 14 {a} + 15 {a} +} { + do_execsql_test 13.1.$rowid { + INSERT INTO t1(rowid, a) VALUES($rowid, $a); + } +} + +#------------------------------------------------------------------------- +# +reset_db +fts5_aux_test_functions db +do_execsql_test 14.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c d'); +} {} + +do_execsql_test 14.1 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {0.0.1} + +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(1, 'sqlite'); + INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); + COMMIT; +} {} + +do_test 15.1 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +do_test 15.2 { + execsql { DELETE FROM t1 } +} {} + +do_execsql_test 15.3.1 { + SELECT rowid FROM t1('sqlite'); +} {} + +do_execsql_test 15.3.2 { + SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC; +} {} + +do_test 15.4 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +finish_test + diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test new file mode 100644 index 0000000000..96cacb293f --- /dev/null +++ b/ext/fts5/test/fts5synonym2.test @@ -0,0 +1,154 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym2 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Code for a simple Tcl tokenizer that supports synonyms at query time. +# +foreach SYNDICT { + {zero 0} + {one 1 i} + {two 2 ii} + {three 3 iii} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} +} { + foreach s $SYNDICT { + set o [list] + foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags == "query"} { + foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + } + } +} + +proc tcl_create {args} { + return "tcl_tokenize" +} + +# +# End of tokenizer code. +#------------------------------------------------------------------------- + +foreach_detail_mode $testprefix { + +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); + INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); + INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); + INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); + INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); + INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); + INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4'); + INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4'); + INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii'); + INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five'); + INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine'); + INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven'); + INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three'); + INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9'); + INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight'); + INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven'); + INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four'); + INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2'); + INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six'); + INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi'); + INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi'); + INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii'); + INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii'); + INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4'); + INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii'); + INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4'); + INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii'); + INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four'); + INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five'); + INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one'); + INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8'); + INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i'); + INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9'); + INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five'); + INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6'); + INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4'); + INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv'); + INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine'); + INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five'); + INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi'); + INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii'); + INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two'); + INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one'); + INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii'); + INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4'); + INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii'); + INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii'); + INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4'); + INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii'); + INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one'); +} + +foreach {tn expr} { + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" + 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" + 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" + + 2.1 "one OR two OR three OR four" + 2.2 "(one AND two) OR (three AND four)" + 2.3 "(one AND two) OR (three AND four) NOT five" + 2.4 "(one AND two) NOT 6" + + 3.1 "b:one AND a:two" + 3.2 "b:one OR a:two" + 3.3 "a:one OR b:1 OR {a b} : i" + + 4.1 "NEAR(one two, 2)" + 4.2 "NEAR(one two three, 2)" + 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" +} { + if {[fts5_expr_ok $expr ss]==0} { + do_test 1.$tn.OMITTED { list } [list] + continue + } + + set res [fts5_query_data $expr ss ASC ::syn] + breakpoint + do_execsql_test 1.$tn.[llength $res].asc { + SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + } $res +} + +} + +finish_test + diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index dc5099c6e3..823b2b2872 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -21,9 +21,44 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + +proc null_list_entries {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc star_from_row {L} { + if {[detail_is_full]==0} { + set L [null_list_entries 2 3 $L] + } + return $L +} + +proc star_from_col {L} { + if {[detail_is_col]} { + set L [null_list_entries 3 4 $L] + } + if {[detail_is_none]} { + set L [null_list_entries 1 4 $L] + set L [null_list_entries 3 4 $L] + } + return $L +} + +proc row_to_col {L} { + if {[detail_is_none]==0} { error "this is for detail=none mode" } + set ret [list] + foreach {a b c} $L { + lappend ret $a {} $b {} + } + set ret +} do_execsql_test 1.1.1 { - CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); PRAGMA table_info = v1; } { @@ -52,32 +87,32 @@ do_execsql_test 1.3 { do_execsql_test 1.4.1 { SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] do_execsql_test 1.4.2 { SELECT * FROM v2; -} {x one 2 4 y one 1 1 z one 1 1} +} [star_from_col {x one 2 4 y one 1 1 z one 1 1}] do_execsql_test 1.5.1 { BEGIN; INSERT INTO t1 VALUES('a b c'); SELECT * FROM v1 WHERE term<'d'; -} {a 1 1 b 1 1 c 1 1} +} [star_from_row {a 1 1 b 1 1 c 1 1}] do_execsql_test 1.5.2 { SELECT * FROM v2 WHERE term<'d'; COMMIT; -} {a one 1 1 b one 1 1 c one 1 1} +} [star_from_col {a one 1 1 b one 1 1 c one 1 1}] do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE tt USING fts5(a, b); + CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); @@ -90,7 +125,12 @@ do_execsql_test 2.0 { INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); } -set res_col { +set res_row [star_from_row { + a 10 20 b 9 14 c 9 20 d 9 19 + e 8 13 f 10 20 g 7 14 x 1 1 + y 1 1 +}] +set res_col [star_from_col { a a 6 11 a b 7 9 b a 6 7 b b 7 7 c a 6 12 c b 5 8 @@ -99,11 +139,9 @@ set res_col { f a 9 10 f b 7 10 g a 5 7 g b 5 7 x a 1 1 y b 1 1 -} -set res_row { - a 10 20 b 9 14 c 9 20 d 9 19 - e 8 13 f 10 20 g 7 14 x 1 1 - y 1 1 +}] +if {[detail_is_none]} { + set res_col [row_to_col $res_row] } foreach {tn tbl resname} { @@ -153,9 +191,9 @@ reset_db forcedelete test.db2 do_execsql_test 5.0 { ATTACH 'test.db2' AS aux; - CREATE VIRTUAL TABLE t1 USING fts5(x); - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); - CREATE VIRTUAL TABLE aux.t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO main.t1 VALUES('a b c'); INSERT INTO main.t1 VALUES('d e f'); @@ -178,18 +216,18 @@ do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); } -do_execsql_test 5.2 { SELECT * FROM vm } { +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 -} -do_execsql_test 5.3 { SELECT * FROM vt1 } { +}] +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.4 { SELECT * FROM vt2 } { +}] +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.5 { SELECT * FROM va } { +}] +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 -} +}] #------------------------------------------------------------------------- # @@ -218,7 +256,7 @@ do_catchsql_test 6.2 { # constraints in the WHERE clause). # do_execsql_test 7.0 { - CREATE VIRTUAL TABLE tx USING fts5(one, two); + CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); @@ -276,6 +314,10 @@ foreach {term} { if {[lindex $r2 2]==0} {set r2 [list]} set resc [concat $r1 $r2] + + set resc [star_from_col $resc] + set resr [star_from_row $resr] + if {[detail_is_none]} { set resc [row_to_col $resr] } do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } @@ -340,10 +382,14 @@ do_execsql_test 7.3.1 { SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; } {30} -do_execsql_test 7.3.2 { - SELECT count(*) FROM txc, txc_c - WHERE txc.term = txc_c.term AND txc.col=txc_c.col; -} {57} +if {![detail_is_none]} { + do_execsql_test 7.3.2 { + SELECT count(*) FROM txc, txc_c + WHERE txc.term = txc_c.term AND txc.col=txc_c.col; + } {57} +} + +} finish_test diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl new file mode 100644 index 0000000000..1b060ea759 --- /dev/null +++ b/ext/fts5/tool/fts5speed.tcl @@ -0,0 +1,59 @@ + + +set Q { + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"} + {25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"} + {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"} + + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} +} + +proc usage {} { + global Q + puts stderr "Usage: $::argv0 DATABASE QUERY" + puts stderr "" + for {set i 1} {$i <= [llength $Q]} {incr i} { + puts stderr " $i. [lindex $Q [expr $i-1]]" + } + puts stderr "" + exit -1 +} + + +set nArg [llength $argv] +if {$nArg!=2 && $nArg!=3} usage +set database [lindex $argv 0] +set iquery [lindex $argv 1] +if {$iquery<1 || $iquery>[llength $Q]} usage +set nRepeat 0 +if {$nArg==3} { set nRepeat [lindex $argv 2] } + + +sqlite3 db $database +catch { load_static_extension db fts5 } + +incr iquery -1 +set sql [lindex $Q $iquery 1] +if {$nRepeat==0} { + set nRepeat [lindex $Q $iquery 0] +} + +puts "sql: $sql" +puts "nRepeat: $nRepeat" +if {[regexp matchinfo $sql]} { + sqlite3_fts5_register_matchinfo db + db eval $sql +} else { + puts "result: [db eval $sql]" +} + +for {set i 1} {$i < $nRepeat} {incr i} { + db eval $sql +} + + diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index dd46743def..018296b8dd 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -75,7 +75,7 @@ int main(int argc, char **argv){ /* Process command line arguments. Following this block local variables ** zTarget, zRbu and nStep are all set. */ if( argc==5 ){ - int nArg1 = strlen(argv[1]); + size_t nArg1 = strlen(argv[1]); if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); nStep = atoi(argv[2]); }else if( argc!=3 ){ @@ -103,7 +103,7 @@ int main(int argc, char **argv){ "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", nProgress ); - fprintf(stdout, zBuf); + fprintf(stdout, "%s", zBuf); break; case SQLITE_DONE: @@ -111,7 +111,7 @@ int main(int argc, char **argv){ "SQLITE_DONE: rbu update completed (%lld operations)\n", nProgress ); - fprintf(stdout, zBuf); + fprintf(stdout, "%s", zBuf); break; default: @@ -122,4 +122,3 @@ int main(int argc, char **argv){ sqlite3_free(zErrmsg); return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1; } - diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 4c38e14c9e..145b446d30 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -935,7 +935,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ p->rc = SQLITE_NOMEM; }else{ @@ -981,8 +981,8 @@ static char *rbuStrndup(const char *zStr, int *pRc){ assert( *pRc==SQLITE_OK ); if( zStr ){ - int nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc(nCopy); + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); if( zRet ){ memcpy(zRet, zStr, nCopy); }else{ @@ -2330,7 +2330,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; RbuFrame *aNew; - aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); if( aNew==0 ) return SQLITE_NOMEM; pRbu->aFrame = aNew; pRbu->nFrameAlloc = nNew; @@ -2395,7 +2395,7 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ if( nChar==0 ){ return 0; } - zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); + zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } @@ -3029,11 +3029,12 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ){ sqlite3rbu *p; - int nTarget = strlen(zTarget); - int nRbu = strlen(zRbu); - int nState = zState ? strlen(zState) : 0; + size_t nTarget = strlen(zTarget); + size_t nRbu = strlen(zRbu); + size_t nState = zState ? strlen(zState) : 0; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; - p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1); + p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ RbuState *pState = 0; @@ -3170,7 +3171,7 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ int i; - int nErrmsg = strlen(p->zErrmsg); + size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ int nDel = 8; @@ -3634,7 +3635,7 @@ static int rbuVfsShmMap( if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); - char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); if( apNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -3645,7 +3646,7 @@ static int rbuVfsShmMap( } if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ - char *pNew = (char*)sqlite3_malloc(szRegion); + char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -3755,7 +3756,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = strlen(zName); + int n = (int)strlen(zName); const char *z = &zName[n]; if( flags & SQLITE_OPEN_URI ){ int odd = 0; @@ -3781,8 +3782,8 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - int nCopy = strlen(zName); - char *zCopy = sqlite3_malloc(nCopy+2); + size_t nCopy = strlen(zName); + char *zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ memcpy(zCopy, zName, nCopy); zCopy[nCopy-3] = 'o'; @@ -4011,13 +4012,13 @@ int sqlite3rbu_create_vfs(const char *zName, const char *zParent){ }; rbu_vfs *pNew = 0; /* Newly allocated VFS */ - int nName; int rc = SQLITE_OK; + size_t nName; + size_t nByte; - int nByte; nName = strlen(zName); nByte = sizeof(rbu_vfs) + nName + 1; - pNew = (rbu_vfs*)sqlite3_malloc(nByte); + pNew = (rbu_vfs*)sqlite3_malloc64(nByte); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/manifest b/manifest index f98559986f..c13c8d974c 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Merge\srecent\senhancements,\sand\sespecially\sthe\sWAL\soverwrite\schange,\sfrom\strunk. -D 2016-01-11T13:10:41.337 -F Makefile.in e3c1e13b4eba06cbb7f9295148383b7e284596be +C Merge\sthe\slatest\sfixes\sand\senhancements\sfrom\strunk. +D 2016-01-14T14:48:17.709 +F Makefile.in c61147c9afcc7a058a3a88876a5b9344fce55353 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 6862a51bfd3716bbe3c387c724feb73937b107d2 +F Makefile.msc d35026ada75f21433cd34be90dc3865972cea6df F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 089e5ecdb5761e64ea1013ded02feb4d8b29927d F autoconf/README 14458f1046c118efa721aadec5f227e876d3cd38 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/configure.ac 82c628c68f0f204d57343dc345b6dbd1789fb1bd +F autoconf/configure.ac 2f41055b10873cfe443a08185343e38c47e2f9f0 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -29,8 +29,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 42b71ad3fe21c9e88fa59e8458ca1a6bc72eb0c0 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure b4519bb54fff37e7cde9d03d8f2946f18d5d0086 x -F configure.ac fcfc67b323d32daaa3e46cf7782d9465ed423a6d +F configure d57d3b9d5c66549e54a4a2960de9813142d30a5a x +F configure.ac c59513d560b3107995c73b9cc1e55bfd086c4764 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 @@ -96,39 +96,39 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9 -F ext/fts5/fts5Int.h acf968e43d57b6b1caf7554d34ec35d6ed3b4fe8 +F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 +F ext/fts5/fts5Int.h 6c5a332e6add01dd69166a252d1818fb75c42a08 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e -F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2 -F ext/fts5/fts5_config.c 0ee66188609a62342e9f9aeefa3c3e44518a4dd6 -F ext/fts5/fts5_expr.c 8228aca3e9af626a1ca9b093d4d24b4f2488ad23 -F ext/fts5/fts5_hash.c 25838d525e97f8662ff3504be94d0bad24f9a37e -F ext/fts5/fts5_index.c 578f46697080f11a1e26cd45a1c039c043a3111d -F ext/fts5/fts5_main.c e11b525778e6fce416d916bfa96926bd1ce4616d -F ext/fts5/fts5_storage.c 57c636d87cbb829d6147c8c8bf78fa53b9ebb526 -F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd -F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf +F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 +F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217 +F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56 +F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 +F ext/fts5/fts5_index.c aa798d3a839847fd351b3d0f49520f190e57c2e3 +F ext/fts5/fts5_main.c 488ceecdb4400ecc6a3d3b2247cedef153955388 +F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8 +F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb +F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 +F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d -F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 -F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c -F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63 -F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe -F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a -F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 -F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c -F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 +F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1 +F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 +F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b +F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced +F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa +F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 +F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c +F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071 +F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5 +F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 -F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test a1b7b6393376bc2adc216527a28f5ae5594069df +F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9 +F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 -F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca +F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb @@ -139,10 +139,11 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 -F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d +F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d +F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 +F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 @@ -150,10 +151,11 @@ F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603 F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b +F ext/fts5/test/fts5fault8.test aeb4717b7b293678bc4d2f3c0159206a525375d9 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 -F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 +F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c @@ -168,15 +170,18 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 -F ext/fts5/test/fts5simple.test 9bded45827b4ab8933c87b7b3bcc3cd47f7378a4 +F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 +F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 +F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0 +F ext/fts5/test/fts5vocab.test e4b12f238f113795615ba6343b63fb326d6a360e +F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221 F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -203,7 +208,7 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/rbu/rbu.c e572f7ddef2ef3a73d03e7b44d36448e466772b7 +F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 F ext/rbu/rbu1.test 57601977588603e82700a43c279bd55282ffa482 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 @@ -225,7 +230,7 @@ F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c ea47de615e911b3a69a8e7fb3be3866298403a25 +F ext/rbu/sqlite3rbu.c bea954197524631f2691ec272e8a42df8cad01cc F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -296,9 +301,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc F src/bitvec.c 1a78d450a17c5016710eec900bedfc5729bf9bdf F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 -F src/btree.c 6bd9b3d778a023e2238a81cd0b87b00085220e0e +F src/btree.c 5d93e2477acb99d50a8b045f2e26a0be3d7751fe F src/btree.h 68ef301795e00cdf1d3ab93abc44a43b7fe771e0 -F src/btreeInt.h b5f2651b41808f038dee9282c5dc0232ce6532d3 +F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 F src/build.c 9d497ff4bf3c82cecb520436e0e9963785627583 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f @@ -318,7 +323,7 @@ F src/insert.c a2302c96652242b2d871445c9975a7a849316dcf F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b -F src/main.c bb8aaa94c50e0c66e32dd158bcbf425ccf071864 +F src/main.c a88a45514be79e5f48b5cf4a147956f267425fa7 F src/malloc.c 8f787669e79de26efc42272b5797bc00fff527c6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -337,11 +342,11 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 82986e1e75782b54da7822dca42d36d974fc2948 +F src/os_unix.c b509b49b40a269e7b75ab511b6e92b2dc9444359 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 58d2593612acb6b542de6715b4af397ea1fa0a35 -F src/pager.h bf25005b4656cd805af43487c3139fca9678d0cc +F src/pager.c f4e9ac39fbb1e0fde97af85c0f4e00eb90764b67 +F src/pager.h 1c2a49143dfba9e69cc8159ef019f472ed8d260b F src/parse.y caad1e98edeca6960493d0c60d31b76820dd7776 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 @@ -353,9 +358,9 @@ F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c d84c091185bc160c349e8bf460ebd084dbd77e64 -F src/shell.c 40ded7e3ade5f2cc89ab1180994f6a6975ebc6f3 -F src/sqlite.h.in 686be87fcbaea46b1aa8197a17b7a437d39764fc +F src/select.c 5b0f2aa9f73ec7b65d1711d485471854d5bad23c +F src/shell.c 01e109c27300379b1c35b254cd294cde635f0179 +F src/sqlite.h.in bb645ccc4090a6e95e7b5d92818f46e41033d076 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d F src/sqliteInt.h 09f70fe715a74ea0048ae05526f2a1b6e3a3b6a2 @@ -418,21 +423,21 @@ F src/update.c d8d675aa299336ac086ad2039d7e812cd6237db0 F src/utf.c 32d7f82aa921322f3e1c956f4b58f019ebd2c6b3 F src/util.c e802e8e311a0d6c48cd1b3e89db164f6f0248d70 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c a479a4bd02b6b77b7ff3ff84825fccc0b967fe6a +F src/vdbe.c 9b4da74dbc1554045c69dacbbceb373272d2e0be F src/vdbe.h bfe3f80dba435377cdb64fd917f2529f0f48ab77 F src/vdbeInt.h 1bff4effc71888e3f304e2e6ac7484e39ab78c28 F src/vdbeapi.c ab2cb8fe23fb9f3195f1311eaa800495d83b6118 -F src/vdbeaux.c 77b4a5cbde74f7c3b1d7052c5c4afea89fb20341 +F src/vdbeaux.c 30f329b90af08d92739185788c6fbfd1c233b8b4 F src/vdbeblob.c cc13eca96b8ec51b6248de785a1aec5df11f5805 F src/vdbemem.c 25b6cfd665b5073480452426e84136edd94140c0 -F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d +F src/vdbesort.c 0971557e5d3c289e46f56a52aed2197c13251de7 F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 -F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 92ca9e7923c337c497e4c9aa7edac800e269a1d8 -F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b -F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c c6d3d2f6af57d574a7365ee2b225a5024f2a6bec +F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 +F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54 +F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c +F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 +F src/where.c bb69654f841ae7af0a20cc6fb8f0ac57901c31be F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a F src/wherecode.c dfbfe198e418b01f208b489e088edd230c91a4e7 F src/whereexpr.c eebba8340c90de73b3d3bbe8c43b84559b8e6e2c @@ -923,7 +928,7 @@ F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 F test/offset1.test f06b83657bcf26f9ce805e67450e189e282143b2 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 870e150450437d3980badbde3d0166b81d9e33f6 +F test/orderby1.test 4d22a7c75f6a83fc1f188cc7bb5192285fdf2552 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1322,7 +1327,7 @@ F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 -F test/waloverwrite.test 8702964967c2f28204f0b4f27af65da81999fc99 +F test/waloverwrite.test a0d2ae0783187374c1e6a9571e0916152977cb81 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 @@ -1417,7 +1422,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 653db1a9294e0b34849c4504599c282a18d8b8c6 +F tool/sqldiff.c 5a26205111e6fa856d9b1535b1637744dcdb930b F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 @@ -1428,7 +1433,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fa4705c91f9650ecd7ec967dbbf0028aabd8a98c 8e807bfaa197027d0cb73532baa96755ce71ea12 -R 207c595909cfacc40e672adb0642fe8a +P c4a858b228a164be2f89f5b01833f0b5e0d7735b 8dedff3b9ac3e6bf9c131fee19f7d26dc1ebd61f +R 99385ae0fdf1facdff7c072dfc1f2a57 U drh -Z c05d49256eed75a98851d063f39abe2d +Z c131c7ae38d871cc572c248d8ac0f193 diff --git a/manifest.uuid b/manifest.uuid index 252d7d9b65..448e42f7f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4a858b228a164be2f89f5b01833f0b5e0d7735b \ No newline at end of file +007e5c6df60f9743ac6914332f59925e4a7a861c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 77552d5831..b5a1ee3b2c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1079,7 +1079,6 @@ static void btreeParseCellPtrNoPayload( ){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 ); - assert( pPage->noPayload ); assert( pPage->childPtrSize==4 ); #ifndef SQLITE_DEBUG UNUSED_PARAMETER(pPage); @@ -1101,8 +1100,6 @@ static void btreeParseCellPtr( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - assert( pPage->intKeyLeaf || pPage->noPayload ); - assert( pPage->noPayload==0 ); assert( pPage->intKeyLeaf ); assert( pPage->childPtrSize==0 ); pIter = pCell; @@ -1171,7 +1168,6 @@ static void btreeParseCellPtrIndex( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf==0 ); - assert( pPage->noPayload==0 ); pIter = pCell + pPage->childPtrSize; nPayload = *pIter; if( nPayload>=0x80 ){ @@ -1232,7 +1228,6 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif - assert( pPage->noPayload==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -1690,11 +1685,9 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->intKey = 1; if( pPage->leaf ){ pPage->intKeyLeaf = 1; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtr; }else{ pPage->intKeyLeaf = 0; - pPage->noPayload = 1; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; } @@ -1709,7 +1702,6 @@ static int decodeFlags(MemPage *pPage, int flagByte){ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; diff --git a/src/btreeInt.h b/src/btreeInt.h index a196e18a28..6cd090257c 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -276,7 +276,6 @@ struct MemPage { u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ - u8 noPayload; /* True if internal intKey page (thus w/o data) */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ diff --git a/src/main.c b/src/main.c index 2652bfc5c9..b37cec8b15 100644 --- a/src/main.c +++ b/src/main.c @@ -3445,6 +3445,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ }else if( op==SQLITE_FCNTL_VFS_POINTER ){ *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ diff --git a/src/os_unix.c b/src/os_unix.c index a40a866780..5bd6116945 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -430,7 +430,11 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) +#if defined(HAVE_FCHOWN) { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, @@ -464,7 +468,11 @@ static struct unix_syscall { #endif #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) +#if defined(HAVE_READLINK) { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) @@ -477,10 +485,10 @@ static struct unix_syscall { ** we are not running as root. */ static int robustFchown(int fd, uid_t uid, gid_t gid){ -#if OS_VXWORKS - return 0; -#else +#if defined(HAVE_FCHOWN) return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; #endif } @@ -5947,6 +5955,7 @@ static int unixFullPathname( assert( pVfs->mxPathname==MAX_PATHNAME ); UNUSED_PARAMETER(pVfs); +#if defined(HAVE_READLINK) /* Attempt to resolve the path as if it were a symbolic link. If it is ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if ** the identified file is not a symbolic link or does not exist, then @@ -5962,6 +5971,7 @@ static int unixFullPathname( }else{ zOut[nByte] = '\0'; } +#endif /* If buffer zOut[] now contains an absolute path there is nothing more ** to do. If it contains a relative path, do the following: diff --git a/src/pager.c b/src/pager.c index 2c8dceb750..2c904d2df1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5627,7 +5627,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( rc!=SQLITE_OK ){ return rc; } - sqlite3WalExclusiveMode(pPager->pWal, 1); + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to @@ -6692,6 +6692,18 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } +/* +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. +*/ +sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif +} + /* ** Return the full pathname of the journal file. */ diff --git a/src/pager.h b/src/pager.h index ba4eec438d..3552a876e7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,7 @@ int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); +sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); diff --git a/src/select.c b/src/select.c index b1db07d09a..fd094d05dd 100644 --- a/src/select.c +++ b/src/select.c @@ -54,6 +54,7 @@ struct SortCtx { int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -124,6 +125,9 @@ Select *sqlite3SelectNew( pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; +#if SELECTTRACE_ENABLED + pNew->zSelName[0] = 0; +#endif pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; @@ -513,6 +517,7 @@ static void pushOntoSorter( int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); assert( nData==1 || regData==regOrigData ); @@ -523,6 +528,9 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ @@ -531,7 +539,6 @@ static void pushOntoSorter( if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -566,6 +573,10 @@ static void pushOntoSorter( pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); @@ -576,14 +587,8 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( pSelect->iLimit ){ + if( iLimit ){ int addr; - int iLimit; - if( pSelect->iOffset ){ - iLimit = pSelect->iOffset+1; - }else{ - iLimit = pSelect->iLimit; - } addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); @@ -1187,7 +1192,7 @@ static void generateSortTail( SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; @@ -1206,6 +1211,7 @@ static void generateSortTail( struct ExprList_item *aOutEx = p->pEList->a; #endif + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeGoto(v, addrBreak); diff --git a/src/shell.c b/src/shell.c index 74b3990705..00893e955f 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1941,6 +1941,7 @@ static char zHelp[] = ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" ".vfsinfo ?AUX? Information about the top-level VFS\n" + ".vfslist List all available VFSes\n" ".vfsname ?AUX? Print the name of the VFS stack\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" " Negative values right-justify\n" @@ -4451,6 +4452,24 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + sqlite3_vfs *pVfs; + sqlite3_vfs *pCurrent = 0; + if( p->db ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); + } + for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ + utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + if( pVfs->pNext ){ + raw_printf(p->out, "-----------------------------------\n"); + } + } + }else + if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ac2e07c67a..53f7085008 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -794,8 +794,13 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database -** connection. See the [sqlite3_file_control()] documentation for -** additional information. +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +**
  • [[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** **
  • [[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. @@ -1010,6 +1015,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_ZIPVFS 25 #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE diff --git a/src/vdbe.c b/src/vdbe.c index 88a8836f14..8e29c5dfa8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2384,7 +2384,6 @@ case OP_Column: { u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ - u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ p2 = pOp->p2; @@ -2562,10 +2561,31 @@ case OP_Column: { assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); assert( t==pC->aType[p2] ); + pDest->enc = encoding; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); + zData = pC->aRow + aOffset[p2]; + if( t<12 ){ + sqlite3VdbeSerialGet(zData, t, pDest); + }else{ + /* If the column value is a string, we need a persistent value, not + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). + */ + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; + pDest->n = len = (t-12)/2; + if( pDest->szMalloc < len+2 ){ + pDest->flags = MEM_Null; + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; + }else{ + pDest->z = pDest->zMalloc; + } + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = aFlag[t&1]; + } }else{ /* This branch happens only when content is on overflow pages */ if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 @@ -2577,38 +2597,20 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. NULL will work for the value for strings - ** and blobs and whatever is in the payloadSize64 variable - ** will work for everything else. */ - sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); + ** content from disk. */ + static u8 aZero[8]; /* This is the bogus content */ + sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, pDest); - if( rc!=SQLITE_OK ){ - goto op_column_error; + if( rc==SQLITE_OK ){ + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; } - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - pDest->flags &= ~MEM_Ephem; } } - pDest->enc = encoding; op_column_out: - /* If the column value is an ephemeral string, go ahead and persist - ** that string in case the cursor moves before the column value is - ** used. The following code does the equivalent of Deephemeralize() - ** but does it faster. */ - if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ - fx = pDest->flags & (MEM_Str|MEM_Blob); - assert( fx!=0 ); - zData = (const u8*)pDest->z; - len = pDest->n; - if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; - memcpy(pDest->z, zData, len); - pDest->z[len] = 0; - pDest->z[len+1] = 0; - pDest->flags = fx|MEM_Term; - } op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2bddeaeea3..b1646f6109 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -306,8 +306,7 @@ int sqlite3VdbeAddOp4Dup8( */ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; - int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); - sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); } @@ -806,7 +805,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - freeP4(db, pOp->p4type, pOp->p4.p); + if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif @@ -868,16 +867,34 @@ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** ** If addr<0 then change P4 on the most recently inserted instruction. */ +static void SQLITE_NOINLINE vdbeChangeP4Full( + Vdbe *p, + Op *pOp, + const char *zP4, + int n +){ + if( pOp->p4type ){ + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = 0; + pOp->p4.p = 0; + } + if( n<0 ){ + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 || db->mallocFailed ){ - if( n!=P4_VTAB ){ - freeP4(db, n, (void*)*(char**)&zP4); - } + assert( p->aOp!=0 || db->mallocFailed ); + if( db->mallocFailed ){ + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); return; } assert( p->nOp>0 ); @@ -886,43 +903,20 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED - || pOp->p4type==P4_INT32 - || pOp->p4type==P4_KEYINFO ); - freeP4(db, pOp->p4type, pOp->p4.p); - pOp->p4.p = 0; + if( n>=0 || pOp->p4type ){ + vdbeChangeP4Full(p, pOp, zP4, n); + return; + } if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; - }else if( zP4==0 ){ - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - }else if( n==P4_KEYINFO ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_KEYINFO; -#ifdef SQLITE_ENABLE_CURSOR_HINTS - }else if( n==P4_EXPR ){ - /* Responsibility for deleting the Expr tree is handed over to the - ** VDBE by this operation. The caller should have already invoked - ** sqlite3ExprDup() or whatever other routine is needed to make a - ** private copy of the tree. */ - pOp->p4.pExpr = (Expr*)zP4; - pOp->p4type = P4_EXPR; -#endif - }else if( n==P4_VTAB ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_VTAB; - sqlite3VtabLock((VTable *)zP4); - assert( ((VTable *)zP4)->db==p->db ); - }else if( n<0 ){ + }else if( zP4!=0 ){ + assert( n<0 ); pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); } } diff --git a/src/vdbesort.c b/src/vdbesort.c index 54e538fd50..e095f80912 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -737,7 +737,7 @@ static int vdbePmaReaderInit( rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ + u64 nByte = 0; /* Size of PMA in bytes */ rc = vdbePmaReadVarint(pReadr, &nByte); pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; diff --git a/src/vxworks.h b/src/vxworks.h index 45a44453a7..60c41a19b8 100644 --- a/src/vxworks.h +++ b/src/vxworks.h @@ -26,4 +26,6 @@ #else /* This is not VxWorks. */ #define OS_VXWORKS 0 +#define HAVE_FCHOWN 1 +#define HAVE_READLINK 1 #endif /* defined(_WRS_KERNEL) */ diff --git a/src/wal.c b/src/wal.c index ae6a54849a..7cf2cabffb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -708,6 +708,8 @@ static void walEncodeFrame( sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); + }else{ + memset(&aFrame[8], 0, 16); } } @@ -3405,4 +3407,10 @@ int sqlite3WalFramesize(Wal *pWal){ } #endif +/* Return the sqlite3_file object for the WAL file +*/ +sqlite3_file *sqlite3WalFile(Wal *pWal){ + return pWal->pWalFd; +} + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 94a049493e..97e6ab4f10 100644 --- a/src/wal.h +++ b/src/wal.h @@ -44,6 +44,7 @@ # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 +# define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -138,5 +139,8 @@ void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); int sqlite3WalFramesize(Wal *pWal); #endif +/* Return the sqlite3_file object for the WAL file */ +sqlite3_file *sqlite3WalFile(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ diff --git a/src/walker.c b/src/walker.c index 81e0f2cd60..1e0ad32871 100644 --- a/src/walker.c +++ b/src/walker.c @@ -36,9 +36,8 @@ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; - if( pExpr==0 ) return WRC_Continue; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); @@ -54,6 +53,9 @@ int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ } return rc & WRC_Abort; } +int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; +} /* ** Call sqlite3WalkExpr() for every expression in list p or until diff --git a/src/where.c b/src/where.c index e86e26ef1a..2cb8334ffb 100644 --- a/src/where.c +++ b/src/where.c @@ -4318,8 +4318,7 @@ WhereInfo *sqlite3WhereBegin( Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, - SQLITE_INT_TO_PTR(n), P4_INT32); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS diff --git a/test/orderby1.test b/test/orderby1.test index 3e785c54e4..831936ae96 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -528,4 +528,21 @@ do_test 8.3 { set res } 5000 +#--------------------------------------------------------------------------- +# https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029 +# +# Adverse interaction between scalar subqueries and the partial-sorting +# logic. +# +do_execsql_test 9.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(y); + INSERT INTO t2 VALUES(9),(8),(3),(4); + SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y); +} {13} + + finish_test diff --git a/test/waloverwrite.test b/test/waloverwrite.test index 5e51a70d3f..aa1154fb97 100644 --- a/test/waloverwrite.test +++ b/test/waloverwrite.test @@ -154,7 +154,7 @@ foreach {tn xtra} { execsql { SELECT sum(length(y)) FROM t1 } db2 } [expr 20*798] - do_test 1.$tn.9 { + do_test 1.$tn.10 { execsql { PRAGMA integrity_check } db2 } ok db2 close diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 56ff53ee25..ae01cd3c4d 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -994,7 +994,7 @@ static int rbuDeltaCreate( zDelta += lenOut; putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; - return zDelta - zOrigDelta; + return (int)(zDelta - zOrigDelta); } /* Compute the hash table used to locate matching sections in the @@ -1141,7 +1141,7 @@ static int rbuDeltaCreate( putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; sqlite3_free(collide); - return zDelta - zOrigDelta; + return (int)(zDelta - zOrigDelta); } /*