From b12dc84fbb12b6137175ce6ed4a72c7927ffc466 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 17 Dec 2015 20:36:13 +0000 Subject: [PATCH 01/44] Add the "offsets=0" option to fts5, to create a smaller index without term offset information. A few things are currently broken on this branch. FossilOrigin-Name: 40b5bbf02a824ca73b33aa4ae1c7d5f65b7cda10 --- ext/fts5/fts5Int.h | 10 ++++- ext/fts5/fts5_buffer.c | 74 ++++++++++++++++++++++++++++++++++ ext/fts5/fts5_config.c | 12 +++++- ext/fts5/fts5_hash.c | 30 +++++++++----- ext/fts5/fts5_index.c | 56 +++++++++++++++++++++---- ext/fts5/fts5_storage.c | 46 +++++++++++++++------ ext/fts5/test/fts5offsets.test | 59 +++++++++++++++++++++++++++ manifest | 28 +++++++------ manifest.uuid | 2 +- 9 files changed, 272 insertions(+), 45 deletions(-) create mode 100644 ext/fts5/test/fts5offsets.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 83a71723ff..436ae0712e 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -151,6 +151,7 @@ struct Fts5Config { char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bOffsets; /* "offsets=" option value (dflt==1) */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; @@ -292,6 +293,13 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); + +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ +typedef struct Fts5Termset Fts5Termset; +int sqlite3Fts5TermsetNew(Fts5Termset**); +int sqlite3Fts5TermsetAdd(Fts5Termset*, const char*, int, int *pbPresent); +void sqlite3Fts5TermsetFree(Fts5Termset*); + /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ @@ -492,7 +500,7 @@ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ -int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize); +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); void sqlite3Fts5HashFree(Fts5Hash*); int sqlite3Fts5HashWrite( diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index e9aab4622a..7e3e137996 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -292,3 +292,77 @@ int sqlite3Fts5IsBareword(char t){ } +/************************************************************************* +*/ +typedef struct Fts5TermsetEntry Fts5TermsetEntry; +struct Fts5TermsetEntry { + char *pTerm; + int nTerm; + Fts5TermsetEntry *pNext; +}; + +struct Fts5Termset { + Fts5TermsetEntry *apHash[512]; +}; + +int sqlite3Fts5TermsetNew(Fts5Termset **pp){ + int rc = SQLITE_OK; + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); + return rc; +} + +int sqlite3Fts5TermsetAdd( + Fts5Termset *p, + const char *pTerm, int nTerm, + int *pbPresent +){ + int rc = SQLITE_OK; + int i; + int hash = 13; + Fts5TermsetEntry *pEntry; + + /* Calculate a hash value for this term */ + for(i=0; iapHash); + + *pbPresent = 0; + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->nTerm==nTerm && memcmp(pEntry->pTerm, pTerm, nTerm)==0 ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } + } + + return rc; +} + +void sqlite3Fts5TermsetFree(Fts5Termset *p){ + if( p ){ + int i; + for(i=0; iapHash); i++){ + Fts5TermsetEntry *pEntry = p->apHash[i]; + while( pEntry ){ + Fts5TermsetEntry *pDel = pEntry; + pEntry = pEntry->pNext; + sqlite3_free(pDel); + } + } + sqlite3_free(p); + } +} + + + diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 6b0e2b28b6..b72fd9b745 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -14,7 +14,6 @@ */ - #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 @@ -345,6 +344,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed offsets=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bOffsets = (zArg[0]=='1'); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } @@ -500,6 +509,7 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; + pRet->bOffsets = 1; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index f184957af6..dc5adadc8e 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -26,6 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; struct Fts5Hash { + int bOffsets; /* Copy of Fts5Config.bOffsets */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ @@ -79,7 +80,7 @@ struct Fts5HashEntry { /* ** Allocate a new hash table. */ -int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; @@ -90,6 +91,7 @@ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; + pNew->bOffsets = pConfig->bOffsets; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; @@ -214,6 +216,7 @@ int sqlite3Fts5HashWrite( Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ + int bNew = pHash->bOffsets; /* If non-delete entry should be written */ /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); @@ -250,6 +253,7 @@ int sqlite3Fts5HashWrite( p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; + p->iCol = (pHash->bOffsets-1); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -286,24 +290,32 @@ int sqlite3Fts5HashWrite( p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iSzPoslist = p->nData; p->nData += 1; - p->iCol = 0; + p->iCol = (pHash->bOffsets-1); p->iPos = 0; p->iRowid = iRowid; + bNew = 1; } if( iCol>=0 ){ /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ - pPtr[p->nData++] = 0x01; - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); - p->iCol = iCol; - p->iPos = 0; + if( pHash->bOffsets==0 ){ + bNew = 1; + p->iCol = iPos = iCol; + }else{ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = iCol; + p->iPos = 0; + } } - /* Append the new position offset */ - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); - p->iPos = iPos; + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c11abda5ba..bf765d0556 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4001,6 +4001,14 @@ struct PoslistCallbackCtx { int eState; /* See above */ }; +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + /* ** TODO: Make this more efficient! */ @@ -4012,6 +4020,28 @@ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +static void fts5PoslistOffsetsCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } + } + } +} + static void fts5PoslistFilterCallback( Fts5Index *p, void *pContext, @@ -4079,12 +4109,20 @@ static void fts5SegiterPoslist( if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ - PoslistCallbackCtx sCtx; - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - sCtx.eState = fts5IndexColsetTest(pColset, 0); - assert( sCtx.eState==0 || sCtx.eState==1 ); - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + if( p->pConfig->bOffsets==0 ){ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + }else{ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + assert( sCtx.eState==0 || sCtx.eState==1 ); + sCtx.eState = fts5IndexColsetTest(pColset, 0); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + } } } } @@ -4446,7 +4484,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ - p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ @@ -4804,7 +4842,9 @@ int sqlite3Fts5IterPoslist( Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + if( pIter->pIndex->pConfig->bOffsets + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset==0 || pIter->bFiltered ){ *pn = pSeg->nPos; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 5fd5dc51a2..c84b213fdf 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -825,6 +825,7 @@ struct Fts5IntegrityCtx { int iCol; int szCol; u64 cksum; + Fts5Termset *pTermset; Fts5Config *pConfig; }; @@ -832,21 +833,33 @@ struct Fts5IntegrityCtx { ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( - void *pContext, /* Pointer to Fts5InsertCtx object */ + void *pContext, /* Pointer to Fts5IntegrityCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ + int rc = SQLITE_OK; Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } - pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken - ); - return SQLITE_OK; + + if( pCtx->pTermset ){ + int bPresent = 0; + rc = sqlite3Fts5TermsetAdd(pCtx->pTermset, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexCksum( + pCtx->pConfig, pCtx->iRowid, 0, pCtx->iCol, pToken, nToken + ); + } + }else{ + pCtx->cksum ^= sqlite3Fts5IndexCksum( + pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken + ); + } + return rc; } /* @@ -886,17 +899,24 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_column_text(pScan, i+1), - sqlite3_column_bytes(pScan, i+1), - (void*)&ctx, - fts5StorageIntegrityCallback - ); - if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + if( pConfig->bOffsets==0 ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + (const char*)sqlite3_column_text(pScan, i+1), + sqlite3_column_bytes(pScan, i+1), + (void*)&ctx, + fts5StorageIntegrityCallback + ); + } + if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; } if( rc!=SQLITE_OK ) break; } diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5offsets.test new file mode 100644 index 0000000000..cd9bfa0054 --- /dev/null +++ b/ext/fts5/test/fts5offsets.test @@ -0,0 +1,59 @@ +# 2015 December 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5offsets + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0); + + INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 + INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 + INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 + INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 + INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 + INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 + INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 + INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8 + INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9 + INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10 +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +foreach {tn match res} { + 1 "a:a" {9} + 2 "b:g" {1 4 6} + 3 "c:h" {1 3 6 8 9 10} +} { + do_execsql_test 1.2.$tn.1 { + SELECT rowid FROM t1($match); + } $res + + do_execsql_test 1.2.$tn.2 { + SELECT rowid FROM t1($match || '*'); + } $res +} + +finish_test + diff --git a/manifest b/manifest index 799ac619df..8e1565e355 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sspellfix1_scriptcode()\sfunction\sto\signore\swhitespace\sand\spunctuation,\nand\sto\srecognize\shebrew\sand\sarabic\sscripts. -D 2015-12-17T14:18:21.904 +C Add\sthe\s"offsets=0"\soption\sto\sfts5,\sto\screate\sa\ssmaller\sindex\swithout\sterm\soffset\sinformation.\sA\sfew\sthings\sare\scurrently\sbroken\son\sthis\sbranch. +D 2015-12-17T20:36:13.853 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,15 +97,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9 -F ext/fts5/fts5Int.h acf968e43d57b6b1caf7554d34ec35d6ed3b4fe8 +F ext/fts5/fts5Int.h 4e1bb66d8e607bf38e881eb455cdf36cc3fa9e42 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 -F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2 -F ext/fts5/fts5_config.c 0ee66188609a62342e9f9aeefa3c3e44518a4dd6 +F ext/fts5/fts5_buffer.c 389d377d04f6e622644c3343ab5e511f6646de36 +F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 F ext/fts5/fts5_expr.c 80075fa45091bad42100c4a5c4f2efc83e43e3af -F ext/fts5/fts5_hash.c 25838d525e97f8662ff3504be94d0bad24f9a37e -F ext/fts5/fts5_index.c 578f46697080f11a1e26cd45a1c039c043a3111d +F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 +F ext/fts5/fts5_index.c 53b3a8f1c9c1f6e5e896b6dc0a7ad26c2eea23a2 F ext/fts5/fts5_main.c ef04699949ab8e42d590ae30188afef7ad58776e -F ext/fts5/fts5_storage.c 9ea3d92178743758b6c54d9fe8836bbbdcc92e3b +F ext/fts5/fts5_storage.c 0dc37a6183e1061e255f23971198d8878159d4ef F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e @@ -156,6 +156,7 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc +F ext/fts5/test/fts5offsets.test 09fc61d553ae4e985afc0146ec77f3439503fc6b F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -1405,7 +1406,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 85ebd46c701e0a094a4690cd8f1d0cbae9aa257c -R 70cdc580d7d9b92b032d33f5e0d50f2e -U drh -Z 6d4b5ed9cd1870281d7d851922f323d8 +P 7adfa4a5794e47f97491c08abeaaac90e826b331 +R a68b4412544bee6f6bc95a23674c55d4 +T *branch * fts5-offsets +T *sym-fts5-offsets * +T -sym-trunk * +U dan +Z a51f39853c3a5371cd0ec3358f50f2d7 diff --git a/manifest.uuid b/manifest.uuid index eebc618153..b6243ea89e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7adfa4a5794e47f97491c08abeaaac90e826b331 \ No newline at end of file +40b5bbf02a824ca73b33aa4ae1c7d5f65b7cda10 \ No newline at end of file From c58b9eeaaa858ac0c7d4b58d67780c73e5ec2ede Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Dec 2015 19:07:14 +0000 Subject: [PATCH 02/44] Fix a problem with prefix queries on fts5 offsets=0 tables. FossilOrigin-Name: ad0987d83c252dd8d6a69321893629d7be805c28 --- ext/fts5/fts5_expr.c | 19 ++++++++++++++----- ext/fts5/fts5_index.c | 11 +++++++---- ext/fts5/test/fts5offsets.test | 12 +++++++++++- manifest | 19 ++++++++----------- manifest.uuid | 2 +- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index eb167b4407..b7bc13b289 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1805,11 +1805,20 @@ Fts5ExprNode *sqlite3Fts5ParseNode( for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 - && pNear->apPhrase[0]->nTerm==1 - && pNear->apPhrase[0]->aTerm[0].pSynonym==0 - ){ - pRet->eType = FTS5_TERM; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ + if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ + pRet->eType = FTS5_TERM; + } + }else if( pParse->pConfig->bOffsets==0 ){ + assert( pParse->rc==SQLITE_OK ); + pParse->rc = SQLITE_ERROR; + assert( pParse->zErr==0 ); + pParse->zErr = sqlite3_mprintf( + "fts5: %s queries are not supported (offsets=0)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3_free(pRet); + pRet = 0; } }else{ fts5ExprAddChildren(pRet, pLeft); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index bf765d0556..5aa87d500f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4119,8 +4119,8 @@ static void fts5SegiterPoslist( PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; - assert( sCtx.eState==0 || sCtx.eState==1 ); sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); } } @@ -4192,8 +4192,8 @@ static int fts5AppendPoslist( assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + if( p->pConfig->bOffsets + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; @@ -4238,13 +4238,13 @@ static int fts5AppendPoslist( } } } - } } return 0; } + static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; @@ -4390,6 +4390,9 @@ static void fts5MergePrefixLists( } } +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ Fts5Buffer tmp = *p1; *p1 = *p2; diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5offsets.test index cd9bfa0054..405824abf9 100644 --- a/ext/fts5/test/fts5offsets.test +++ b/ext/fts5/test/fts5offsets.test @@ -22,9 +22,11 @@ ifcapable !fts5 { } +#-------------------------------------------------------------------------- +# Simple tests. +# do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0); - INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 @@ -55,5 +57,13 @@ foreach {tn match res} { } $res } +do_catchsql_test 1.3.1 { + SELECT rowid FROM t1('h + d'); +} {1 {fts5: phrase queries are not supported (offsets=0)}} + +do_catchsql_test 1.3.2 { + SELECT rowid FROM t1('NEAR(h d)'); +} {1 {fts5: NEAR queries are not supported (offsets=0)}} + finish_test diff --git a/manifest b/manifest index 8e1565e355..1beb3af7bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"offsets=0"\soption\sto\sfts5,\sto\screate\sa\ssmaller\sindex\swithout\sterm\soffset\sinformation.\sA\sfew\sthings\sare\scurrently\sbroken\son\sthis\sbranch. -D 2015-12-17T20:36:13.853 +C Fix\sa\sproblem\swith\sprefix\squeries\son\sfts5\soffsets=0\stables. +D 2015-12-18T19:07:14.984 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -101,9 +101,9 @@ F ext/fts5/fts5Int.h 4e1bb66d8e607bf38e881eb455cdf36cc3fa9e42 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 389d377d04f6e622644c3343ab5e511f6646de36 F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 -F ext/fts5/fts5_expr.c 80075fa45091bad42100c4a5c4f2efc83e43e3af +F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2 F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 -F ext/fts5/fts5_index.c 53b3a8f1c9c1f6e5e896b6dc0a7ad26c2eea23a2 +F ext/fts5/fts5_index.c d862dc84c54133d902893a6a7992699cd5272c48 F ext/fts5/fts5_main.c ef04699949ab8e42d590ae30188afef7ad58776e F ext/fts5/fts5_storage.c 0dc37a6183e1061e255f23971198d8878159d4ef F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd @@ -156,7 +156,7 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc -F ext/fts5/test/fts5offsets.test 09fc61d553ae4e985afc0146ec77f3439503fc6b +F ext/fts5/test/fts5offsets.test 8410ec485d652da168c138b3d39e7418ff8406bf F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -1406,10 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7adfa4a5794e47f97491c08abeaaac90e826b331 -R a68b4412544bee6f6bc95a23674c55d4 -T *branch * fts5-offsets -T *sym-fts5-offsets * -T -sym-trunk * +P 40b5bbf02a824ca73b33aa4ae1c7d5f65b7cda10 +R 3f45bfddac5bb4e5875c9115c2804e36 U dan -Z a51f39853c3a5371cd0ec3358f50f2d7 +Z 1ef5385fc9e0304197471eb155fb346c diff --git a/manifest.uuid b/manifest.uuid index b6243ea89e..af1abb33df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40b5bbf02a824ca73b33aa4ae1c7d5f65b7cda10 \ No newline at end of file +ad0987d83c252dd8d6a69321893629d7be805c28 \ No newline at end of file From 159fd77e0f6ea30171b64471036d73d721f1271a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 21 Dec 2015 18:45:09 +0000 Subject: [PATCH 03/44] Fix an fts5 integrity-check problem that affects offsets=0 tables with prefix indexes. FossilOrigin-Name: 609a0bc7f34e6dae74ce756aff920f3df78fe828 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_buffer.c | 57 ++++++++++++++++++++-------------- ext/fts5/fts5_index.c | 49 +++++++++-------------------- ext/fts5/fts5_storage.c | 41 +++++++++++++++++------- ext/fts5/test/fts5ad.test | 16 ++++++++++ ext/fts5/test/fts5auto.test | 1 - ext/fts5/test/fts5offsets.test | 16 +++++++++- ext/fts5/test/fts5simple.test | 12 ++++++- manifest | 26 ++++++++-------- manifest.uuid | 2 +- 10 files changed, 133 insertions(+), 89 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 436ae0712e..3016f4be04 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -297,7 +297,7 @@ int sqlite3Fts5IsBareword(char t); /* Bucket of terms object used by the integrity-check in offsets=0 mode. */ typedef struct Fts5Termset Fts5Termset; int sqlite3Fts5TermsetNew(Fts5Termset**); -int sqlite3Fts5TermsetAdd(Fts5Termset*, const char*, int, int *pbPresent); +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); void sqlite3Fts5TermsetFree(Fts5Termset*); /* diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 7e3e137996..251a543c5a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -298,6 +298,7 @@ typedef struct Fts5TermsetEntry Fts5TermsetEntry; struct Fts5TermsetEntry { char *pTerm; int nTerm; + int iIdx; /* Index (main or aPrefix[] entry) */ Fts5TermsetEntry *pNext; }; @@ -313,36 +314,44 @@ int sqlite3Fts5TermsetNew(Fts5Termset **pp){ int sqlite3Fts5TermsetAdd( Fts5Termset *p, + int iIdx, const char *pTerm, int nTerm, int *pbPresent ){ int rc = SQLITE_OK; - int i; - int hash = 13; - Fts5TermsetEntry *pEntry; - - /* Calculate a hash value for this term */ - for(i=0; iapHash); - *pbPresent = 0; - for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ - if( pEntry->nTerm==nTerm && memcmp(pEntry->pTerm, pTerm, nTerm)==0 ){ - *pbPresent = 1; - break; - } - } + if( p ){ + int i; + int hash; + Fts5TermsetEntry *pEntry; - if( pEntry==0 ){ - pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); - if( pEntry ){ - pEntry->pTerm = (char*)&pEntry[1]; - pEntry->nTerm = nTerm; - memcpy(pEntry->pTerm, pTerm, nTerm); - pEntry->pNext = p->apHash[hash]; - p->apHash[hash] = pEntry; + /* Calculate a hash value for this term */ + hash = 104 + iIdx; + for(i=0; iapHash); + + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + ){ + *pbPresent = 1; + break; + } + } + + if( pEntry==0 ){ + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); + if( pEntry ){ + pEntry->pTerm = (char*)&pEntry[1]; + pEntry->nTerm = nTerm; + pEntry->iIdx = iIdx; + memcpy(pEntry->pTerm, pTerm, nTerm); + pEntry->pNext = p->apHash[hash]; + p->apHash[hash] = pEntry; + } } } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 5aa87d500f..bcd81843b7 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4373,7 +4373,9 @@ static void fts5MergePrefixLists( sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); } } - p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); + if( iNew!=writer.iPrev || tmp.n==0 ){ + p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); + } } /* WRITEPOSLISTSIZE */ @@ -4608,7 +4610,11 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ -static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){ +static int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +){ int n = 0; int i; for(i=0; inPrefix && rc==SQLITE_OK; i++){ - int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); + const int nChar = pConfig->aPrefix[i]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, @@ -4983,7 +4990,7 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ /* ** Return a simple checksum value based on the arguments. */ -static u64 fts5IndexEntryCksum( +u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, @@ -5071,7 +5078,7 @@ static int fts5QueryCksum( ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); - cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } rc = sqlite3Fts5IterNext(pIdxIter); } @@ -5370,7 +5377,7 @@ static void fts5IndexIntegrityCheckSegment( /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums -** as calculated by fts5IndexEntryCksum() is cksum. +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without @@ -5434,7 +5441,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); - cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); @@ -5450,34 +5457,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ return fts5IndexReturn(p); } - -/* -** Calculate and return a checksum that is the XOR of the index entry -** checksum of all entries that would be generated by the token specified -** by the final 5 arguments. -*/ -u64 sqlite3Fts5IndexCksum( - Fts5Config *pConfig, /* Configuration object */ - i64 iRowid, /* Document term appears in */ - int iCol, /* Column term appears in */ - int iPos, /* Position term appears in */ - const char *pTerm, int nTerm /* Term at iPos */ -){ - u64 ret = 0; /* Return value */ - int iIdx; /* For iterating through indexes */ - - ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); - - for(iIdx=0; iIdxnPrefix; iIdx++){ - int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); - if( nByte ){ - ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); - } - } - - return ret; -} - /************************************************************************* ************************************************************************** ** Below this point is the implementation of the fts5_decode() scalar diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index c84b213fdf..65429edbd9 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -829,6 +829,7 @@ struct Fts5IntegrityCtx { Fts5Config *pConfig; }; + /* ** Tokenization callback used by integrity check. */ @@ -840,25 +841,41 @@ static int fts5StorageIntegrityCallback( int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ - int rc = SQLITE_OK; Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; + Fts5Termset *pTermset = pCtx->pTermset; + int bPresent; + int ii; + int rc = SQLITE_OK; + int iPos; + int iCol; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } - if( pCtx->pTermset ){ - int bPresent = 0; - rc = sqlite3Fts5TermsetAdd(pCtx->pTermset, pToken, nToken, &bPresent); - if( rc==SQLITE_OK && bPresent==0 ){ - pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, 0, pCtx->iCol, pToken, nToken - ); - } - }else{ - pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken + iPos = pTermset ? pCtx->iCol : pCtx->szCol-1; + iCol = pTermset ? 0 : pCtx->iCol; + + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); + if( rc==SQLITE_OK && bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, 0, pToken, nToken ); } + + for(ii=0; rc==SQLITE_OK && iipConfig->nPrefix; ii++){ + const int nChar = pCtx->pConfig->aPrefix[ii]; + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); + if( nByte ){ + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); + if( bPresent==0 ){ + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte + ); + } + } + } + return rc; } diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index 3881c7e161..a9fc568133 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -74,6 +74,22 @@ foreach {T create} { BEGIN; } + 6 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + + 7 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + + 8 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + } + } { do_test $T.1 { diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test index 771a0b64d8..6efaa9cb7c 100644 --- a/ext/fts5/test/fts5auto.test +++ b/ext/fts5/test/fts5auto.test @@ -369,7 +369,6 @@ foreach {tn expr} { } { -breakpoint do_auto_test 4.$tn yy {c1 c2 c3} $expr } diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5offsets.test index 405824abf9..b70bee3b53 100644 --- a/ext/fts5/test/fts5offsets.test +++ b/ext/fts5/test/fts5offsets.test @@ -21,7 +21,6 @@ ifcapable !fts5 { return } - #-------------------------------------------------------------------------- # Simple tests. # @@ -65,5 +64,20 @@ do_catchsql_test 1.3.2 { SELECT rowid FROM t1('NEAR(h d)'); } {1 {fts5: NEAR queries are not supported (offsets=0)}} +#------------------------------------------------------------------------- +# integrity-check with both offsets= and prefix= options. +# +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1"); + INSERT INTO t2(a) VALUES('aa ab'); +} + +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} + +breakpoint +do_execsql_test 2.1 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + finish_test diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 27a1aee08b..870a912bb1 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -18,7 +18,7 @@ ifcapable !fts5 { finish_test return } - + #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" @@ -350,6 +350,16 @@ do_execsql_test 4.1 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' } {0 {} 4} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 15.0 { + CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); + INSERT INTO x2 VALUES('ab'); +} + +do_execsql_test 15.1 { + INSERT INTO x2(x2) VALUES('integrity-check'); +} finish_test diff --git a/manifest b/manifest index 1beb3af7bd..34b04180c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sprefix\squeries\son\sfts5\soffsets=0\stables. -D 2015-12-18T19:07:14.984 +C Fix\san\sfts5\sintegrity-check\sproblem\sthat\saffects\soffsets=0\stables\swith\sprefix\sindexes. +D 2015-12-21T18:45:09.329 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,15 +97,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9 -F ext/fts5/fts5Int.h 4e1bb66d8e607bf38e881eb455cdf36cc3fa9e42 +F ext/fts5/fts5Int.h e71739ed0b816758f1f77baaccca1b4a9064c4dc F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 -F ext/fts5/fts5_buffer.c 389d377d04f6e622644c3343ab5e511f6646de36 +F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2 F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 -F ext/fts5/fts5_index.c d862dc84c54133d902893a6a7992699cd5272c48 +F ext/fts5/fts5_index.c 1d1939afbc434907993d7e9a0c631be630d5b0dc F ext/fts5/fts5_main.c ef04699949ab8e42d590ae30188afef7ad58776e -F ext/fts5/fts5_storage.c 0dc37a6183e1061e255f23971198d8878159d4ef +F ext/fts5/fts5_storage.c 14e0bb6549a66da54adf4fa1564edbf24647cb22 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e @@ -118,7 +118,7 @@ F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c -F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63 +F ext/fts5/test/fts5ad.test 21d87b12c7ec83b4ec48816d24503443dffb10a1 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 @@ -128,7 +128,7 @@ F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 F ext/fts5/test/fts5al.test a1b7b6393376bc2adc216527a28f5ae5594069df F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 -F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca +F ext/fts5/test/fts5auto.test 2a6241673657b340427f521528f7809ddaa02a9e F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb @@ -156,7 +156,7 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc -F ext/fts5/test/fts5offsets.test 8410ec485d652da168c138b3d39e7418ff8406bf +F ext/fts5/test/fts5offsets.test 1a2d53c34a896d2038b839df2178410c45977671 F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -169,7 +169,7 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 -F ext/fts5/test/fts5simple.test 9bded45827b4ab8933c87b7b3bcc3cd47f7378a4 +F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 40b5bbf02a824ca73b33aa4ae1c7d5f65b7cda10 -R 3f45bfddac5bb4e5875c9115c2804e36 +P ad0987d83c252dd8d6a69321893629d7be805c28 +R f519655f5c64b0aca45be5b21174cea1 U dan -Z 1ef5385fc9e0304197471eb155fb346c +Z 77d1a35391efdb0fe1eff3fc97431cfb diff --git a/manifest.uuid b/manifest.uuid index af1abb33df..32831e991d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad0987d83c252dd8d6a69321893629d7be805c28 \ No newline at end of file +609a0bc7f34e6dae74ce756aff920f3df78fe828 \ No newline at end of file From 8f9ced636f61773bc67a1405d3ed89b1d22356fb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 22 Dec 2015 18:54:16 +0000 Subject: [PATCH 04/44] Updates so that fts5 API functions xInst, xPhraseFirst and xPhraseNext work with the offsets=0 option. FossilOrigin-Name: 69bffc1632c8a8f3bfe5bf92607e64fed982e48c --- ext/fts5/fts5.h | 10 +++- ext/fts5/fts5_main.c | 16 +++++++ ext/fts5/fts5_tcl.c | 31 +++++++++++++ ext/fts5/fts5_test_mi.c | 2 +- ext/fts5/test/fts5_common.tcl | 13 ++++++ ext/fts5/test/fts5offsets.test | 85 +++++++++++++++++++++++++++++++++- manifest | 22 ++++----- manifest.uuid | 2 +- 8 files changed, 165 insertions(+), 16 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 5f528af793..400b7c7885 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -110,6 +110,12 @@ struct Fts5PhraseIter { ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. The exception is if the table was created +** with the offsets=0 option specified. In this case *piOff is always +** set to -1. +** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** @@ -196,7 +202,7 @@ struct Fts5PhraseIter { ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); -** iOff>=0; +** iCol>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol @@ -210,7 +216,7 @@ struct Fts5PhraseIter { ** See xPhraseFirst above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 1 */ + int iVersion; /* Currently always set to 2 */ void *(*xUserData)(Fts5Context*); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index cf2c748c4c..1d0b4c219e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -309,6 +309,13 @@ static int fts5IsContentless(Fts5Table *pTab){ return pTab->pConfig->eContent==FTS5_CONTENT_NONE; } +/* +** Return true if pTab is an offsetless table. +*/ +static int fts5IsOffsetless(Fts5Table *pTab){ + return pTab->pConfig->bOffsets==0; +} + /* ** Delete a virtual table handle allocated by fts5InitVtab(). */ @@ -1749,6 +1756,10 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; + }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ + *piPhrase = pCsr->aInst[iIdx*3]; + *piCol = pCsr->aInst[iIdx*3 + 2]; + *piOff = -1; }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -1914,6 +1925,11 @@ static void fts5ApiPhraseNext( if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; + }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){ + int iVal; + pIter->a += fts5GetVarint32(pIter->a, iVal); + *piCol += (iVal-2); + *piOff = -1; }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index b470c557d3..bf815eab81 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -235,6 +235,7 @@ static int xF5tApi( { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ + { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { 0, 0, 0} }; @@ -431,6 +432,36 @@ static int xF5tApi( break; } + CASE(16, "xPhraseForeach") { + int iPhrase; + int iCol; + int iOff; + const char *zColvar; + const char *zOffvar; + Tcl_Obj *pScript = objv[5]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + zOffvar = Tcl_GetString(objv[4]); + + for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); + iCol>=0; + p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) + ){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 355f23330d..fb4a2102b4 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -134,7 +134,7 @@ static int fts5MatchinfoXCb( int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); - iOff>=0; + iCol>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 2c64b3b9a4..74e84ab234 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -28,6 +28,18 @@ proc fts5_test_poslist {cmd} { set res } +proc fts5_test_poslist2 {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseForeach $i c o { + lappend res $i.$c.$o + } + } + + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -113,6 +125,7 @@ proc fts5_aux_test_functions {db} { fts5_test_columntext fts5_test_columntotalsize fts5_test_poslist + fts5_test_poslist2 fts5_test_tokenize fts5_test_rowcount fts5_test_all diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5offsets.test index b70bee3b53..2f2674fabe 100644 --- a/ext/fts5/test/fts5offsets.test +++ b/ext/fts5/test/fts5offsets.test @@ -74,10 +74,93 @@ do_execsql_test 2.0 { #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} -breakpoint do_execsql_test 2.1 { INSERT INTO t2(t2) VALUES('integrity-check'); } +#------------------------------------------------------------------------- +# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs +# work with offsets=0 tables. +# +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} + 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} + 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} + 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} + 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} + 6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb} + 7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc} + 8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb} + 9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb} + 10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa} + 11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac} + 12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc} + 13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc} + 14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa} + 15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc} + 16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc} + 17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca} + 18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc} + 19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc} + 20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc} + 21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb} + 22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc} + 23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc} + 24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb} + 25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc} + 26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa} + 27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac} + 28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa} + 29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac} + 30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac} + 31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba} + 32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc} + 33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa} + 34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac} + 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} + 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} + 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} + 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} + 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} + 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} +} +foreach {tn tbl} { + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) } +} { + reset_db + fts5_aux_test_functions db + execsql $tbl + foreach {id x y z} $data { + execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } + } + foreach {tn2 expr} { + 1 aaa 2 ccc 3 bab 4 aac + 5 aa* 6 cc* 7 ba* 8 aa* + 9 a* 10 b* 11 c* + } { + + set res [list] + foreach {id x y z} $data { + if {[lsearch [concat $x $y $z] $expr]>=0} { + lappend res $id + set inst [list] + if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 } + if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 } + if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 } + lappend res $inst + } + } + + do_execsql_test 3.$tn.$tn2.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) + } $res + + do_execsql_test 3.$tn.$tn2.2 { + SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) + } $res + } + +} + finish_test diff --git a/manifest b/manifest index 34b04180c1..d02d515584 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sfts5\sintegrity-check\sproblem\sthat\saffects\soffsets=0\stables\swith\sprefix\sindexes. -D 2015-12-21T18:45:09.329 +C Updates\sso\sthat\sfts5\sAPI\sfunctions\sxInst,\sxPhraseFirst\sand\sxPhraseNext\swork\swith\sthe\soffsets=0\soption. +D 2015-12-22T18:54:16.763 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -96,7 +96,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9 +F ext/fts5/fts5.h 7d6785c75afe23239dad9cbaffc6fc55803feb4b F ext/fts5/fts5Int.h e71739ed0b816758f1f77baaccca1b4a9064c4dc F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 @@ -104,17 +104,17 @@ F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2 F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 F ext/fts5/fts5_index.c 1d1939afbc434907993d7e9a0c631be630d5b0dc -F ext/fts5/fts5_main.c ef04699949ab8e42d590ae30188afef7ad58776e +F ext/fts5/fts5_main.c b7d6540c63f347908d8ee937ca8a7987130c84b3 F ext/fts5/fts5_storage.c 14e0bb6549a66da54adf4fa1564edbf24647cb22 -F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd -F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf +F ext/fts5/fts5_tcl.c c3cfff5f727b3d02f73c5717413d637b18b960f2 +F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d +F ext/fts5/test/fts5_common.tcl 4a23a75f31dbb96c03c2840fae1bf85b4f64f062 F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c @@ -156,7 +156,7 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc -F ext/fts5/test/fts5offsets.test 1a2d53c34a896d2038b839df2178410c45977671 +F ext/fts5/test/fts5offsets.test 7d32df518f6513131c14ce985006c3ac5873e79f F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad0987d83c252dd8d6a69321893629d7be805c28 -R f519655f5c64b0aca45be5b21174cea1 +P 609a0bc7f34e6dae74ce756aff920f3df78fe828 +R 3d59533357e77f6a30434e9a4ea67f85 U dan -Z 77d1a35391efdb0fe1eff3fc97431cfb +Z ae770209a83bcb452cb39fda88b8ef75 diff --git a/manifest.uuid b/manifest.uuid index 32831e991d..07c12c4bf0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -609a0bc7f34e6dae74ce756aff920f3df78fe828 \ No newline at end of file +69bffc1632c8a8f3bfe5bf92607e64fed982e48c \ No newline at end of file From 9f44deed93ea3b94e661d9fcafff41d5d8afa0de Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Dec 2015 19:55:00 +0000 Subject: [PATCH 05/44] Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. FossilOrigin-Name: 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d --- ext/fts5/fts5Int.h | 10 ++- ext/fts5/fts5_config.c | 45 ++++++++-- ext/fts5/fts5_expr.c | 87 ++++++++++++++++++- ext/fts5/fts5_hash.c | 20 +++-- ext/fts5/fts5_index.c | 33 +++---- ext/fts5/fts5_main.c | 77 +++++++++++----- ext/fts5/fts5_storage.c | 2 +- ext/fts5/test/fts5_common.tcl | 18 +++- ext/fts5/test/fts5ac.test | 80 ++++++++++++----- ext/fts5/test/fts5ad.test | 6 +- .../{fts5offsets.test => fts5detail.test} | 87 ++++++++++++++----- manifest | 32 +++---- manifest.uuid | 2 +- 13 files changed, 379 insertions(+), 120 deletions(-) rename ext/fts5/test/{fts5offsets.test => fts5detail.test} (76%) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 3016f4be04..a0f69bee21 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -151,7 +151,7 @@ struct Fts5Config { char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ - int bOffsets; /* "offsets=" option value (dflt==1) */ + int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; @@ -180,6 +180,9 @@ struct Fts5Config { #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 @@ -637,6 +640,11 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int +); + int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index b72fd9b745..fd6da7f11e 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -194,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){ } } + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. @@ -344,12 +371,16 @@ static int fts5ConfigParseSpecial( return rc; } - if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){ - if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ - *pzErr = sqlite3_mprintf("malformed offsets=... directive"); - rc = SQLITE_ERROR; - }else{ - pConfig->bOffsets = (zArg[0]=='1'); + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); } return rc; } @@ -509,7 +540,7 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; - pRet->bOffsets = 1; + pRet->eDetail = FTS5_DETAIL_FULL; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index b7bc13b289..5a5a0652ee 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1744,6 +1744,15 @@ void sqlite3Fts5ParseSetColset( Fts5ExprNearset *pNear, Fts5Colset *pColset ){ + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pParse->rc = SQLITE_ERROR; + pParse->zErr = sqlite3_mprintf( + "fts5: column queries are not supported (detail=none)" + ); + sqlite3_free(pColset); + return; + } + if( pNear ){ pNear->pColset = pColset; }else{ @@ -1809,12 +1818,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode( if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ pRet->eType = FTS5_TERM; } - }else if( pParse->pConfig->bOffsets==0 ){ + }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; assert( pParse->zErr==0 ); pParse->zErr = sqlite3_mprintf( - "fts5: %s queries are not supported (offsets=0)", + "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); sqlite3_free(pRet); @@ -1932,6 +1941,9 @@ static char *fts5ExprPrintTcl( for(iTerm=0; zRet && iTermnTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); @@ -2244,3 +2256,74 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ } return nRet; } + +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ + int i; + Fts5PoslistWriter *pRet; + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + pBuf->n = 0; + } + pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase); + if( pRet ){ + memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase); + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistWriter *aWriter; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +static int fts5ExprPopulatePoslistsCb( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ +){ + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; + Fts5Expr *pExpr = p->pExpr; + int i; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pTerm; + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + int nTerm = strlen(pTerm->zTerm); + if( (nTerm==nToken || (nTermbPrefix)) + && memcmp(pTerm->zTerm, pToken, nTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff + ); + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistWriter *aWriter, + int iCol, + const char *z, int n +){ + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aWriter = aWriter; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index dc5adadc8e..a9f1c13ede 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -26,7 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; struct Fts5Hash { - int bOffsets; /* Copy of Fts5Config.bOffsets */ + int eDetail; /* Copy of Fts5Config.eDetail */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; - pNew->bOffsets = pConfig->bOffsets; + pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; @@ -216,7 +216,9 @@ int sqlite3Fts5HashWrite( Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ - int bNew = pHash->bOffsets; /* If non-delete entry should be written */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); @@ -253,7 +255,7 @@ int sqlite3Fts5HashWrite( p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; - p->iCol = (pHash->bOffsets-1); + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -290,7 +292,7 @@ int sqlite3Fts5HashWrite( p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iSzPoslist = p->nData; p->nData += 1; - p->iCol = (pHash->bOffsets-1); + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->iPos = 0; p->iRowid = iRowid; bNew = 1; @@ -300,14 +302,14 @@ int sqlite3Fts5HashWrite( /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ - if( pHash->bOffsets==0 ){ - bNew = 1; - p->iCol = iPos = iCol; - }else{ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); p->iCol = iCol; p->iPos = 0; + }else{ + bNew = 1; + p->iCol = iPos = iCol; } } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index bcd81843b7..836a6d2c99 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4109,19 +4109,19 @@ static void fts5SegiterPoslist( if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ - if( p->pConfig->bOffsets==0 ){ - PoslistOffsetsCtx sCtx; - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); - }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; sCtx.eState = fts5IndexColsetTest(pColset, 0); assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); } } } @@ -4192,7 +4192,7 @@ static int fts5AppendPoslist( assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - if( p->pConfig->bOffsets + if( p->pConfig->eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ @@ -4850,9 +4850,11 @@ int sqlite3Fts5IterPoslist( i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int eDetail = pIter->pIndex->pConfig->eDetail; + assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - if( pIter->pIndex->pConfig->bOffsets + if( eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; @@ -4871,7 +4873,9 @@ int sqlite3Fts5IterPoslist( }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - *pp = pIter->poslist.p; + if( eDetail==FTS5_DETAIL_FULL ){ + *pp = pIter->poslist.p; + } *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); @@ -5062,17 +5066,15 @@ static int fts5QueryCksum( ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; + Fts5Buffer buf = {0, 0, 0}; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ - i64 dummy; - const u8 *pPos; - int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ @@ -5084,6 +5086,7 @@ static int fts5QueryCksum( } } sqlite3Fts5IterClose(pIdxIter); + fts5BufferFree(&buf); *pCksum = cksum; return rc; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 1d0b4c219e..51c3f57cff 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -226,6 +226,7 @@ struct Fts5Cursor { #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) @@ -313,7 +314,7 @@ static int fts5IsContentless(Fts5Table *pTab){ ** Return true if pTab is an offsetless table. */ static int fts5IsOffsetless(Fts5Table *pTab){ - return pTab->pConfig->bOffsets==0; + return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL; } /* @@ -646,6 +647,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){ FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST ); } @@ -1603,6 +1605,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ return rc; } +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**); + static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; @@ -1652,8 +1656,53 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + } + } + return rc; +} + static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ int n; + int rc = SQLITE_OK; + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistWriter *aWriter; + int i; +assert( pCsr->pSorter==0 ); /* fixme */ + aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); + if( aWriter==0 ) rc = SQLITE_NOMEM; + for(i=0; inCol && rc==SQLITE_OK; i++){ + int n; const char *z; + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aWriter, i, z, n + ); + } + } + sqlite3_free(aWriter); + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); @@ -1756,10 +1805,12 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; +#if 0 }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 2]; *piOff = -1; +#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -1773,27 +1824,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } -static int fts5ApiColumnText( - Fts5Context *pCtx, - int iCol, - const char **pz, - int *pn -){ - int rc = SQLITE_OK; - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ - *pz = 0; - *pn = 0; - }else{ - rc = fts5SeekCursor(pCsr, 0); - if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); - } - } - return rc; -} - static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, @@ -1925,11 +1955,13 @@ static void fts5ApiPhraseNext( if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; +#if 0 }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); *piCol += (iVal-2); *piOff = -1; +#endif }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); @@ -1982,7 +2014,6 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, }; - /* ** Implementation of API function xQueryPhrase(). */ diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 65429edbd9..fe3ce829a1 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -916,7 +916,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - if( pConfig->bOffsets==0 ){ + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 74e84ab234..30b191e299 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -239,7 +239,21 @@ proc nearset {aCol args} { set iFirst [expr $iFL - $O(-near) - [llength $p]] for {set i $iFirst} {$i <= $iFL} {incr i} { - if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i } + set lCand [lrange $col $i [expr $i+$nPm1]] + + set bMatch 1 + foreach tok $p term $lCand { + if {[string match $tok $term]==0} { + #puts "$tok $term failed" + set bMatch 0 + } + } + if {$bMatch} { + #puts "match at $i" + lappend B($iPhrase) $i + } + + #if {$lCand == $p} { lappend B($iPhrase) $i } } if {[llength $B($iPhrase)] == 0} break } @@ -265,7 +279,7 @@ proc nearset {aCol args} { incr counter } - #puts $res + #puts "$aCol -> $res" sort_poslist $res } diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 0de4848145..cc10cece09 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -130,7 +130,7 @@ set data { # # CREATE VIRTUAL TABLE xy USING fts5(x, y); # -# Assuming the table contains the same records as stored int the global +# Assuming the table contains the same records as stored in the global # $::data array (see above), this function returns a list containing one # element for each match in the dataset. The elements are themselves lists # formatted as follows: @@ -175,30 +175,22 @@ proc matchdata {bPos expr {bAsc 1}} { # End of test code #------------------------------------------------------------------------- -proc fts5_test_poslist {cmd} { - set res [list] - for {set i 0} {$i < [$cmd xInstCount]} {incr i} { - lappend res [string map {{ } .} [$cmd xInst $i]] - } - set res -} - foreach {tn2 sql} { 1 {} 2 {BEGIN} } { reset_db - sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist + fts5_aux_test_functions db - do_execsql_test 1.0 { + do_execsql_test 1.$tn2.0 { CREATE VIRTUAL TABLE xx USING fts5(x,y); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } execsql $sql - do_test $tn2.1.1 { + do_test 1.$tn2.1.1 { foreach {id x y} $data { execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } } @@ -224,7 +216,7 @@ foreach {tn2 sql} { set expr "\"$phrase\"" set res [matchdata 1 $expr] - do_execsql_test $tn2.1.2.$tn.[llength $res] { + do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -246,7 +238,7 @@ foreach {tn2 sql} { 3.1 { a AND b AND c } } { set res [matchdata 1 $expr] - do_execsql_test $tn2.2.$tn.[llength $res] { + do_execsql_test 1.$tn2.2.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -275,7 +267,7 @@ foreach {tn2 sql} { 4.4 {{"y" y}:b} } { set res [matchdata 1 $expr] - do_execsql_test $tn2.3.$tn.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -296,7 +288,7 @@ foreach {tn2 sql} { 9 { y : NEAR(r c) } } { set res [matchdata 1 $expr] - do_execsql_test $tn2.4.1.$tn.[llength $res] { + do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -307,14 +299,14 @@ foreach {tn2 sql} { foreach {tn expr tclexpr} { 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} } { - do_execsql_test $tn2.5.$tn { + do_execsql_test 1.$tn2.5.$tn { SELECT fts5_expr_tcl($expr, 'N $x') } [list $tclexpr] } #------------------------------------------------------------------------- # - do_execsql_test $tn2.6.integrity { + do_execsql_test 1.$tn2.6.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} @@ -346,14 +338,62 @@ foreach {tn2 sql} { 19 { c NOT b OR a AND d } } { set res [matchdata 0 $expr $bAsc] - do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res + do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res } } } -do_execsql_test 3.1 { +do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} +# Some tests for detail=col tables and detail=none. +# +foreach {tn2 sql} { + 1 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); + } + 2 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); + BEGIN; + } + 3 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); + BEGIN; + } + 4 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); + } +} { + reset_db + fts5_aux_test_functions db + + execsql $sql + + do_execsql_test 3.$tn2.0 { + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); + } + + + do_test 3.$tn2.1.1 { + foreach {id x y} $data { + execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } + } + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } + } {} + + foreach {tn q} { + 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" + 11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d" + } { + set res [matchdata 1 $q] + + do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q + } $res + } + +} + finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index a9fc568133..cc86a6b2eb 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -75,17 +75,17 @@ foreach {T create} { } 6 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 7 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 8 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5detail.test similarity index 76% rename from ext/fts5/test/fts5offsets.test rename to ext/fts5/test/fts5detail.test index 2f2674fabe..59f4b0e3c3 100644 --- a/ext/fts5/test/fts5offsets.test +++ b/ext/fts5/test/fts5detail.test @@ -13,7 +13,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5offsets +set testprefix fts5detail # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { @@ -21,11 +21,13 @@ ifcapable !fts5 { return } +fts5_aux_test_functions db + #-------------------------------------------------------------------------- # Simple tests. # do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 @@ -58,17 +60,18 @@ foreach {tn match res} { do_catchsql_test 1.3.1 { SELECT rowid FROM t1('h + d'); -} {1 {fts5: phrase queries are not supported (offsets=0)}} +} {1 {fts5: phrase queries are not supported (detail!=full)}} do_catchsql_test 1.3.2 { SELECT rowid FROM t1('NEAR(h d)'); -} {1 {fts5: NEAR queries are not supported (offsets=0)}} +} {1 {fts5: NEAR queries are not supported (detail!=full)}} + #------------------------------------------------------------------------- -# integrity-check with both offsets= and prefix= options. +# integrity-check with both detail= and prefix= options. # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1"); + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); INSERT INTO t2(a) VALUES('aa ab'); } @@ -78,9 +81,17 @@ do_execsql_test 2.1 { INSERT INTO t2(t2) VALUES('integrity-check'); } +do_execsql_test 2.2 { + SELECT fts5_test_poslist(t2) FROM t2('aa'); +} {0.0.0} + +set ::pc 0 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] +#exit + #------------------------------------------------------------------------- # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs -# work with offsets=0 tables. +# work with detail=col tables. # set data { 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} @@ -124,9 +135,42 @@ set data { 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} } + +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} +} + +proc matchdata {expr {bAsc 1}} { + + set tclexpr [db one { + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') + }] + set res [list] + + #puts "$expr -> $tclexpr" + foreach {id x y z} $::data { + set cols [list $x $y $z] + set ::pc 0 + #set hits [lsort -command instcompare [eval $tclexpr]] + set hits [eval $tclexpr] + if {[llength $hits]>0} { + lappend res [list $id $hits] + } + } + + if {$bAsc} { + set res [lsort -integer -increasing -index 0 $res] + } else { + set res [lsort -integer -decreasing -index 0 $res] + } + + return [concat {*}$res] +} + foreach {tn tbl} { - 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) } + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } } { +#break reset_db fts5_aux_test_functions db execsql $tbl @@ -139,17 +183,7 @@ foreach {tn tbl} { 9 a* 10 b* 11 c* } { - set res [list] - foreach {id x y z} $data { - if {[lsearch [concat $x $y $z] $expr]>=0} { - lappend res $id - set inst [list] - if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 } - if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 } - if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 } - lappend res $inst - } - } + set res [matchdata $expr] do_execsql_test 3.$tn.$tn2.1 { SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) @@ -159,8 +193,21 @@ foreach {tn tbl} { SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) } $res } - } +#------------------------------------------------------------------------- +# Simple tests for detail=none tables. +# +breakpoint +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); +} + +do_catchsql_test 4.1 { + SELECT * FROM t4('a:a') +} {1 {fts5: column queries are not supported (detail=none)}} + finish_test diff --git a/manifest b/manifest index d02d515584..050615992a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sso\sthat\sfts5\sAPI\sfunctions\sxInst,\sxPhraseFirst\sand\sxPhraseNext\swork\swith\sthe\soffsets=0\soption. -D 2015-12-22T18:54:16.763 +C Change\sthe\sname\sof\sthe\soffsets=0\soption\sto\s"detail=column".\sHave\sthe\sxInst,\sxPhraseFirst\sand\sother\sAPI\sfunctions\swork\sby\sparsing\sthe\soriginal\stext\sfor\sdetail=column\stables. +D 2015-12-28T19:55:00.739 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,15 +97,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 7d6785c75afe23239dad9cbaffc6fc55803feb4b -F ext/fts5/fts5Int.h e71739ed0b816758f1f77baaccca1b4a9064c4dc +F ext/fts5/fts5Int.h 7328cfa7b0151e34bb3715d404d732c6d49d5361 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 -F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 -F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2 -F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 -F ext/fts5/fts5_index.c 1d1939afbc434907993d7e9a0c631be630d5b0dc -F ext/fts5/fts5_main.c b7d6540c63f347908d8ee937ca8a7987130c84b3 -F ext/fts5/fts5_storage.c 14e0bb6549a66da54adf4fa1564edbf24647cb22 +F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 +F ext/fts5/fts5_expr.c c9522ae99f862ddb318ea69998a02147d2a00c8d +F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40 +F ext/fts5/fts5_index.c 24e81d8f2ce92f50eb19c15bffd5ada4ec91d8de +F ext/fts5/fts5_main.c a71db65929e150031c5a195e3c782e773deb55a6 +F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179 F ext/fts5/fts5_tcl.c c3cfff5f727b3d02f73c5717413d637b18b960f2 F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e @@ -114,11 +114,11 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 4a23a75f31dbb96c03c2840fae1bf85b4f64f062 +F ext/fts5/test/fts5_common.tcl 27f941596e241e25b4bd64c2446ae91dd4cb138c F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c -F ext/fts5/test/fts5ad.test 21d87b12c7ec83b4ec48816d24503443dffb10a1 +F ext/fts5/test/fts5ac.test 5dea9e119ed39e0923b423dc2b3eaaef2ec3abd3 +F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 @@ -139,6 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 +F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 w ext/fts5/test/fts5offsets.test F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 @@ -156,7 +157,6 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc -F ext/fts5/test/fts5offsets.test 7d32df518f6513131c14ce985006c3ac5873e79f F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 609a0bc7f34e6dae74ce756aff920f3df78fe828 -R 3d59533357e77f6a30434e9a4ea67f85 +P 69bffc1632c8a8f3bfe5bf92607e64fed982e48c +R bce3f221a6a9e036b503a3a5522049ea U dan -Z ae770209a83bcb452cb39fda88b8ef75 +Z 73bb0a8c75a5e4585189e64532bdf9a9 diff --git a/manifest.uuid b/manifest.uuid index 07c12c4bf0..a1ac4a70e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69bffc1632c8a8f3bfe5bf92607e64fed982e48c \ No newline at end of file +228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d \ No newline at end of file From 9a3a5275cd9ac4aa6c3c5f63cf289a1aa63d5514 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Dec 2015 19:35:03 +0000 Subject: [PATCH 06/44] Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase. FossilOrigin-Name: 8c30605bcd0a78a5015948171145bc6f640b8358 --- ext/fts5/fts5.h | 7 ++- ext/fts5/fts5Int.h | 4 ++ ext/fts5/fts5_expr.c | 22 ++++++++ ext/fts5/fts5_index.c | 11 ++++ ext/fts5/fts5_main.c | 75 ++++++++++++++++++++++--- ext/fts5/fts5_tcl.c | 27 +++++++++ ext/fts5/test/fts5_common.tcl | 11 ++++ ext/fts5/test/fts5ac.test | 101 ++++++++++++++++++++++++++-------- manifest | 28 +++++----- manifest.uuid | 2 +- 10 files changed, 241 insertions(+), 47 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 400b7c7885..990c3f2f0d 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -216,7 +216,7 @@ struct Fts5PhraseIter { ** See xPhraseFirst above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); @@ -246,8 +246,11 @@ struct Fts5ExtensionApi { int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); - void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index a0f69bee21..4ef99ebae1 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -447,6 +447,8 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -647,6 +649,8 @@ int sqlite3Fts5ExprPopulatePoslists( int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 5a5a0652ee..ebd767a617 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2327,3 +2327,25 @@ int sqlite3Fts5ExprPopulatePoslists( ); } +/* +** This function is only called for detail=columns tables. +*/ +int sqlite3Fts5ExprPhraseCollist( + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, + int *pnCollist +){ + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; + Fts5ExprNode *pNode = pPhrase->pNode; + assert( iPhrase>=0 && iPhrasenPhrase ); + + if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ + sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + }else{ + *ppCollist = 0; + *pnCollist = 0; + } + return SQLITE_OK; +} + diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 836a6d2c99..34caa45072 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4881,6 +4881,17 @@ int sqlite3Fts5IterPoslist( return fts5IndexReturn(pIter->pIndex); } +int sqlite3Fts5IterCollist( + Fts5IndexIter *pIter, + const u8 **pp, /* OUT: Pointer to position-list data */ + int *pn /* OUT: Size of position-list in bytes */ +){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + *pp = pIter->poslist.p; + *pn = pIter->poslist.n; + return SQLITE_OK; +} + /* ** This function is similar to sqlite3Fts5IterPoslist(), except that it ** copies the position list into the buffer supplied as the second diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 51c3f57cff..cd43308513 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1955,13 +1955,6 @@ static void fts5ApiPhraseNext( if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; -#if 0 - }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){ - int iVal; - pIter->a += fts5GetVarint32(pIter->a, iVal); - *piCol += (iVal-2); - *piOff = -1; -#endif }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); @@ -1975,7 +1968,7 @@ static void fts5ApiPhraseNext( } } -static void fts5ApiPhraseFirst( +static int fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, @@ -1989,6 +1982,70 @@ static void fts5ApiPhraseFirst( fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); } +static void fts5ApiPhraseNextColumn( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int *piCol +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pIter->a>=pIter->b ){ + *piCol = -1; + }else{ + int iIncr; + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); + *piCol += (iIncr-2); + } + }else{ + while( 1 ){ + int dummy; + if( pIter->a>=pIter->b ){ + *piCol = -1; + return; + } + if( pIter->a[0]==0x01 ) break; + pIter->a += fts5GetVarint32(pIter->a, dummy); + } + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + } +} + +static int fts5ApiPhraseFirstColumn( + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, + int *piCol +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + int n; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); + } + }else{ + int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); + pIter->b = &pIter->a[n]; + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } + } + + return rc; +} + + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); @@ -2012,6 +2069,8 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, + fts5ApiPhraseFirstColumn, + fts5ApiPhraseNextColumn, }; /* diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index bf815eab81..4a331a59f4 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -236,6 +236,7 @@ static int xF5tApi( { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ + { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ { 0, 0, 0} }; @@ -462,6 +463,32 @@ static int xF5tApi( break; } + CASE(17, "xPhraseColumnForeach") { + int iPhrase; + int iCol; + const char *zColvar; + Tcl_Obj *pScript = objv[4]; + Fts5PhraseIter iter; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + zColvar = Tcl_GetString(objv[3]); + + for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); + iCol>=0; + p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol) + ){ + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc==TCL_CONTINUE ) rc = TCL_OK; + if( rc!=TCL_OK ){ + if( rc==TCL_BREAK ) rc = TCL_OK; + break; + } + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 30b191e299..64af5c59b3 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -40,6 +40,16 @@ proc fts5_test_poslist2 {cmd} { set res } +proc fts5_test_collist {cmd} { + set res [list] + + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { + $cmd xPhraseColumnForeach $i c { lappend res $i.$c } + } + + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -126,6 +136,7 @@ proc fts5_aux_test_functions {db} { fts5_test_columntotalsize fts5_test_poslist fts5_test_poslist2 + fts5_test_collist fts5_test_tokenize fts5_test_rowcount fts5_test_all diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index cc10cece09..ff2affcce1 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -131,16 +131,17 @@ set data { # CREATE VIRTUAL TABLE xy USING fts5(x, y); # # Assuming the table contains the same records as stored in the global -# $::data array (see above), this function returns a list containing one -# element for each match in the dataset. The elements are themselves lists -# formatted as follows: +# $::data array (see above), this function returns a list containing two +# elements for each matching row in the dataset. The first element of each +# pair is the rowid. The second is a list of phrase matches, where each +# phrase match is of the form: # -# { ...} +# .. # -# where each element is a list of phrase matches in the -# same form as returned by auxiliary scalar function fts5_test(). +# The list of phrase matches is in the same format as that returned by the +# fts5_test_poslist() auxiliary scalara function. # -proc matchdata {bPos expr {bAsc 1}} { +proc poslist_data {bPos expr {bAsc 1}} { set tclexpr [db one { SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y') @@ -168,13 +169,43 @@ proc matchdata {bPos expr {bAsc 1}} { set res [lsort -integer -decreasing -index 0 $res] } - return [concat {*}$res] + set res [concat {*}$res] + return $res +} + +proc collist_elem_compare {a b} { + foreach {a1 a2} [split $a .] {} + foreach {b1 b2} [split $b .] {} + + if {$a1==$b1} { + return [expr $a2 - $b2] + } + return [expr $a1 - $b1] +} + +proc poslist2collist {poslist} { + set res [list] + foreach h $poslist { + regexp {(.*)\.[1234567890]+} $h -> cand + lappend res $cand + } + set res [lsort -command collist_elem_compare -unique $res] + return $res +} + +proc collist_data {expr} { + set res [list] + foreach {rowid poslist} [poslist_data 1 $expr] { + lappend res $rowid [poslist2collist $poslist] + } + set res } # # End of test code #------------------------------------------------------------------------- +if 0 { foreach {tn2 sql} { 1 {} @@ -214,11 +245,16 @@ foreach {tn2 sql} { 10 "L O O L V V K" } { set expr "\"$phrase\"" - set res [matchdata 1 $expr] + set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { + do_execsql_test 1.$tn2.1.2.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.1.2.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res } #------------------------------------------------------------------------- @@ -237,10 +273,15 @@ foreach {tn2 sql} { 3.1 { a AND b AND c } } { - set res [matchdata 1 $expr] - do_execsql_test 1.$tn2.2.$tn.[llength $res] { + set res [poslist_data 1 $expr] + do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res } #------------------------------------------------------------------------- @@ -266,10 +307,15 @@ foreach {tn2 sql} { 4.3 {{x "x"}:b} 4.4 {{"y" y}:b} } { - set res [matchdata 1 $expr] - do_execsql_test 1.$tn2.3.$tn.[llength $res] { + set res [poslist_data 1 $expr] + do_execsql_test 1.$tn2.3.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res } #------------------------------------------------------------------------- @@ -287,10 +333,15 @@ foreach {tn2 sql} { 8 { x : NEAR(r c) } 9 { y : NEAR(r c) } } { - set res [matchdata 1 $expr] - do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { + set res [poslist_data 1 $expr] + do_execsql_test 1.$tn2.4.1.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.4.1.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res } do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0} @@ -337,7 +388,7 @@ foreach {tn2 sql} { 18 { c NOT (b OR a) } 19 { c NOT b OR a AND d } } { - set res [matchdata 0 $expr $bAsc] + set res [poslist_data 0 $expr $bAsc] do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res } } @@ -347,6 +398,9 @@ do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} +} +#set data [lrange $data 0 5] + # Some tests for detail=col tables and detail=none. # foreach {tn2 sql} { @@ -374,7 +428,6 @@ foreach {tn2 sql} { INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } - do_test 3.$tn2.1.1 { foreach {id x y} $data { execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } @@ -384,13 +437,17 @@ foreach {tn2 sql} { foreach {tn q} { 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" - 11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d" + 11 "o a" 12 "c AND d" 13 "o OR a" 14 "c OR d" } { - set res [matchdata 1 $q] - - do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { + set res [poslist_data 1 $q] + do_execsql_test 3.$tn2.1.2.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q } $res + + set res [collist_data $q] + do_execsql_test 3.$tn2.1.2.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $q + } $res } } diff --git a/manifest b/manifest index 050615992a..8b22a0fdc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\soffsets=0\soption\sto\s"detail=column".\sHave\sthe\sxInst,\sxPhraseFirst\sand\sother\sAPI\sfunctions\swork\sby\sparsing\sthe\soriginal\stext\sfor\sdetail=column\stables. -D 2015-12-28T19:55:00.739 +C Add\sthe\sxPhraseFirstColumn()\sand\sxPhraseNextColumn()\sAPI\sfunctions\sto\sfts5.\sFor\siterating\sthrough\sthe\sset\sof\scolumns\sthat\scontain\sintances\sof\sa\sphrase. +D 2015-12-29T19:35:03.765 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -96,17 +96,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 7d6785c75afe23239dad9cbaffc6fc55803feb4b -F ext/fts5/fts5Int.h 7328cfa7b0151e34bb3715d404d732c6d49d5361 +F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 +F ext/fts5/fts5Int.h ab5f6f1f65652310a644d97d12aeabf8ab5ed17e F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c c9522ae99f862ddb318ea69998a02147d2a00c8d +F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5 F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40 -F ext/fts5/fts5_index.c 24e81d8f2ce92f50eb19c15bffd5ada4ec91d8de -F ext/fts5/fts5_main.c a71db65929e150031c5a195e3c782e773deb55a6 +F ext/fts5/fts5_index.c ca8310eaa286e6c1e4c4581a420b7c3a1ec6302c +F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89 F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179 -F ext/fts5/fts5_tcl.c c3cfff5f727b3d02f73c5717413d637b18b960f2 +F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c @@ -114,10 +114,10 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 27f941596e241e25b4bd64c2446ae91dd4cb138c +F ext/fts5/test/fts5_common.tcl f923471d572d1e283b7d6985ce3cda62dae6fd45 F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 5dea9e119ed39e0923b423dc2b3eaaef2ec3abd3 +F ext/fts5/test/fts5ac.test 1d2a10d4c13dcfd326287f51df3b7d90761c212c F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a @@ -139,7 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 -F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 w ext/fts5/test/fts5offsets.test +F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 @@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69bffc1632c8a8f3bfe5bf92607e64fed982e48c -R bce3f221a6a9e036b503a3a5522049ea +P 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d +R 9b7a210ea600814406fb8e0040363d54 U dan -Z 73bb0a8c75a5e4585189e64532bdf9a9 +Z 50bb7fa01e78a4726d7b3d5426b2a1ed diff --git a/manifest.uuid b/manifest.uuid index a1ac4a70e8..f3ee4608cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d \ No newline at end of file +8c30605bcd0a78a5015948171145bc6f640b8358 \ No newline at end of file From d11e397b2c0aa6e53209cc46dcf3929252c98243 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Dec 2015 19:58:57 +0000 Subject: [PATCH 07/44] Updates to fts5 to support detail=none mode. As of this commit, many cases are still broken. FossilOrigin-Name: ac8f4cf0cede6bcbe47eeefb85d80a27e9278212 --- ext/fts5/fts5_hash.c | 166 ++++++++++++++++++++------------- ext/fts5/fts5_index.c | 143 +++++++++++++++++++++++----- ext/fts5/test/fts5simple2.test | 75 +++++++++++++++ manifest | 15 +-- manifest.uuid | 2 +- 5 files changed, 309 insertions(+), 92 deletions(-) create mode 100644 ext/fts5/test/fts5simple2.test diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index a9f1c13ede..50ca082711 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -63,6 +63,7 @@ struct Fts5HashEntry { int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ + u8 bContent; /* Set content-flag (detail=none mode) */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ @@ -184,26 +185,46 @@ static int fts5HashResize(Fts5Hash *pHash){ return SQLITE_OK; } -static void fts5HashAddPoslistSize(Fts5HashEntry *p){ +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ if( p->iSzPoslist ){ u8 *pPtr = (u8*)p; - int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ - int nPos = nSz*2 + p->bDel; /* Value of nPos field */ - - assert( p->bDel==0 || p->bDel==1 ); - if( nPos<=127 ){ - pPtr[p->iSzPoslist] = (u8)nPos; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + assert( p->nData==p->iSzPoslist ); + if( p->bDel ){ + pPtr[p->nData++] = 0x00; + if( p->bContent ){ + pPtr[p->nData++] = 0x00; + } + } }else{ - int nByte = sqlite3Fts5GetVarintLen((u32)nPos); - memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); - sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); - p->nData += (nByte-1); + int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ + + assert( p->bDel==0 || p->bDel==1 ); + if( nPos<=127 ){ + pPtr[p->iSzPoslist] = (u8)nPos; + }else{ + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); + p->nData += (nByte-1); + } } - p->bDel = 0; + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; } } +/* +** Add an entry to the in-memory hash table. The key is the concatenation +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). +** +** (bByte || pToken) -> (iRowid,iCol,iPos) +** +** Or, if iCol is negative, then the value is a delete marker. +*/ int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ @@ -233,15 +254,18 @@ int sqlite3Fts5HashWrite( /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ + /* Figure out how much space to allocate */ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; + /* Grow the Fts5Hash.aSlot[] array if necessary. */ if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } + /* Allocate new Fts5HashEntry and add it to the hash table. */ p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); @@ -251,79 +275,95 @@ int sqlite3Fts5HashWrite( assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; - p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); - p->iSzPoslist = p->nData; - p->nData += 1; - p->iRowid = iRowid; - p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; - nIncr += p->nData; - } - /* Check there is enough space to append a new entry. Worst case scenario - ** is: - ** - ** + 9 bytes for a new rowid, - ** + 4 byte reserved for the "poslist size" varint. - ** + 1 byte for a "new column" byte, - ** + 3 bytes for a new column number (16-bit max) as a varint, - ** + 5 bytes for the new position offset (32-bit max). - */ - if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ - int nNew = p->nAlloc * 2; - Fts5HashEntry *pNew; - Fts5HashEntry **pp; - pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->nAlloc = nNew; - for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); - *pp = pNew; - p = pNew; + /* Add the first rowid field to the hash-entry */ + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); + p->iRowid = iRowid; + + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + } + + nIncr += p->nData; + }else{ + + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario + ** is: + ** + ** + 9 bytes for a new rowid, + ** + 4 byte reserved for the "poslist size" varint. + ** + 1 byte for a "new column" byte, + ** + 3 bytes for a new column number (16-bit max) as a varint, + ** + 5 bytes for the new position offset (32-bit max). + */ + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ + int nNew = p->nAlloc * 2; + Fts5HashEntry *pNew; + Fts5HashEntry **pp; + pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->nAlloc = nNew; + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); + *pp = pNew; + p = pNew; + } + nIncr -= p->nData; } + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); + pPtr = (u8*)p; - nIncr -= p->nData; /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); - p->iSzPoslist = p->nData; - p->nData += 1; - p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); - p->iPos = 0; p->iRowid = iRowid; bNew = 1; + p->iSzPoslist = p->nData; + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ + p->nData += 1; + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); + p->iPos = 0; + } } if( iCol>=0 ){ - /* Append a new column value, if necessary */ - assert( iCol>=p->iCol ); - if( iCol!=p->iCol ){ - if( pHash->eDetail==FTS5_DETAIL_FULL ){ - pPtr[p->nData++] = 0x01; - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); - p->iCol = iCol; - p->iPos = 0; - }else{ - bNew = 1; - p->iCol = iPos = iCol; + if( pHash->eDetail==FTS5_DETAIL_NONE ){ + p->bContent = 1; + }else{ + /* Append a new column value, if necessary */ + assert( iCol>=p->iCol ); + if( iCol!=p->iCol ){ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ + pPtr[p->nData++] = 0x01; + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); + p->iCol = iCol; + p->iPos = 0; + }else{ + bNew = 1; + p->iCol = iPos = iCol; + } } - } - /* Append the new position offset, if necessary */ - if( bNew ){ - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); - p->iPos = iPos; + /* Append the new position offset, if necessary */ + if( bNew ){ + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); + p->iPos = iPos; + } } }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; } - nIncr += p->nData; + nIncr += p->nData; *pHash->pnByte += nIncr; return SQLITE_OK; } @@ -437,7 +477,7 @@ int sqlite3Fts5HashQuery( } if( p ){ - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ @@ -473,7 +513,7 @@ void sqlite3Fts5HashScanEntry( Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); - fts5HashAddPoslistSize(p); + fts5HashAddPoslistSize(pHash, p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 34caa45072..e85fb70311 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -452,7 +452,8 @@ struct Fts5SegIter { Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ - int bDel; /* True if the delete flag is set */ + u8 bDel; /* True if the delete flag is set */ + // u8 bContent; /* True if has content (detail=none mode) */ }; /* @@ -466,7 +467,6 @@ struct Fts5SegIter { #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 - /* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or @@ -1492,13 +1492,28 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ - int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); - fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); - pIter->bDel = (nSz & 0x0001); - pIter->nPos = nSz>>1; + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->bDel = 0; + pIter->nPos = 1; + if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + pIter->bDel = 1; + iOff++; + if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + pIter->nPos = 1; + iOff++; + }else{ + pIter->nPos = 0; + } + } + }else{ + int nSz; + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); + } pIter->iLeafOffset = iOff; - assert_nc( pIter->nPos>=0 ); } } @@ -1758,7 +1773,11 @@ static void fts5SegIterNext( int n = pLeaf->szLeaf; ASSERT_SZLEAF_OK(pLeaf); - iOff = pIter->iLeafOffset + pIter->nPos; + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + iOff = pIter->iLeafOffset; + }else{ + iOff = pIter->iLeafOffset + pIter->nPos; + } if( iOffrc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); +#endif + fts5SegIterLoadNPos(p, pIter); } } } @@ -4165,6 +4191,16 @@ static int fts5IndexExtractCol( return p - (*pa); } +static int fts5AppendRowid( + Fts5Index *p, + i64 iDelta, + Fts5IndexIter *pMulti, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); + return 0; +} /* ** Iterator pMulti currently points to a valid entry (not EOF). This @@ -4305,6 +4341,67 @@ static void fts5MergeAppendDocid( (iLastRowid) = (iRowid); \ } +/* +** Swap the contents of buffer *p1 with that of *p2. +*/ +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ + Fts5Buffer tmp = *p1; + *p1 = *p2; + *p2 = tmp; +} + +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ + int i = *piOff; + if( i>=pBuf->n ){ + *piOff = -1; + }else{ + u64 iVal; + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); + *piRowid += iVal; + } +} + +/* +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. +** In this case the buffers consist of a delta-encoded list of rowids only. +*/ +static void fts5MergeRowidLists( + Fts5Index *p, /* FTS5 backend object */ + Fts5Buffer *p1, /* First list to merge */ + Fts5Buffer *p2 /* Second list to merge */ +){ + int i1 = 0; + int i2 = 0; + i64 iRowid1 = 0; + i64 iRowid2 = 0; + i64 iOut = 0; + + Fts5Buffer out; + memset(&out, 0, sizeof(out)); + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); + if( p->rc ) return; + + fts5NextRowid(p1, &i1, &iRowid1); + fts5NextRowid(p1, &i2, &iRowid2); + while( i1>=0 || i2>=0 ){ + if( i1>=0 && (i2<0 || iRowid1=0 && iRowid1==iRowid2 ){ + fts5NextRowid(p1, &i1, &iRowid1); + } + } + } + + fts5BufferSwap(&out, p1); + fts5BufferFree(&out); +} + /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before @@ -4392,15 +4489,6 @@ static void fts5MergePrefixLists( } } -/* -** Swap the contents of buffer *p1 with that of *p2. -*/ -static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ - Fts5Buffer tmp = *p1; - *p1 = *p2; - *p2 = tmp; -} - static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -4413,6 +4501,16 @@ static void fts5SetupPrefixIter( Fts5Buffer *aBuf; const int nBuf = 32; + void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); + int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*); + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + xMerge = fts5MergeRowidLists; + xAppend = fts5AppendRowid; + }else{ + xMerge = fts5MergePrefixLists; + xAppend = fts5AppendPoslist; + } + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); @@ -4445,21 +4543,21 @@ static void fts5SetupPrefixIter( fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ - fts5MergePrefixLists(p, &doclist, &aBuf[i]); + xMerge(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } iLastRowid = 0; } - if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ + if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ iLastRowid = iRowid; } } for(i=0; irc==SQLITE_OK ){ - fts5MergePrefixLists(p, &doclist, &aBuf[i]); + xMerge(p, &doclist, &aBuf[i]); } fts5BufferFree(&aBuf[i]); } @@ -4854,6 +4952,9 @@ int sqlite3Fts5IterPoslist( assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; + if( eDetail==FTS5_DETAIL_NONE ){ + *pn = pSeg->nPos; + }else if( eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test new file mode 100644 index 0000000000..883b27e024 --- /dev/null +++ b/ext/fts5/test/fts5simple2.test @@ -0,0 +1,75 @@ +# 2015 September 05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5simple2 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c'); +} +do_execsql_test 1.1 { + SELECT rowid FROM t1('c a b') +} {1} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); + COMMIT; +} +do_execsql_test 2.1 { + SELECT rowid FROM t1('b c d') +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('b c d'); + INSERT INTO t1 VALUES('b c d'); +} +do_execsql_test 3.1 { + SELECT rowid FROM t1('b c d'); COMMIT; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + BEGIN; + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a3 b3 c3'); + COMMIT; +} +breakpoint +do_execsql_test 4.1 { + SELECT rowid FROM t1('b*'); +} {1 2 3} + + +finish_test + diff --git a/manifest b/manifest index 8b22a0fdc2..64a15ecd6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sxPhraseFirstColumn()\sand\sxPhraseNextColumn()\sAPI\sfunctions\sto\sfts5.\sFor\siterating\sthrough\sthe\sset\sof\scolumns\sthat\scontain\sintances\sof\sa\sphrase. -D 2015-12-29T19:35:03.765 +C Updates\sto\sfts5\sto\ssupport\sdetail=none\smode.\sAs\sof\sthis\scommit,\smany\scases\sare\sstill\sbroken. +D 2015-12-30T19:58:57.068 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -102,8 +102,8 @@ F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5 -F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40 -F ext/fts5/fts5_index.c ca8310eaa286e6c1e4c4581a420b7c3a1ec6302c +F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 +F ext/fts5/fts5_index.c 94dec3c322e9179fb0dd3239d6fe9555b9c621ec F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89 F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb @@ -170,6 +170,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 +F ext/fts5/test/fts5simple2.test 70b98039c8a73341bb4a632adb3018e1017e3beb F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1406,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d -R 9b7a210ea600814406fb8e0040363d54 +P 8c30605bcd0a78a5015948171145bc6f640b8358 +R 393cc400b4b232cee04e2e069dffc8fe U dan -Z 50bb7fa01e78a4726d7b3d5426b2a1ed +Z c9f782e6b3f3ae84d42c40bf56b4157b diff --git a/manifest.uuid b/manifest.uuid index f3ee4608cf..aecac29523 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c30605bcd0a78a5015948171145bc6f640b8358 \ No newline at end of file +ac8f4cf0cede6bcbe47eeefb85d80a27e9278212 \ No newline at end of file From 841243156ff222587f59e1572ffe0f8b20bb42bb Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 31 Dec 2015 17:36:58 +0000 Subject: [PATCH 08/44] Fix some problems with fts5 detail=none tables. Some still remain. FossilOrigin-Name: 6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca --- ext/fts5/fts5_index.c | 51 ++++++++++++++++++------- ext/fts5/test/fts5simple2.test | 70 +++++++++++++++++++++++++++++++++- manifest | 14 +++---- manifest.uuid | 2 +- 4 files changed, 114 insertions(+), 23 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e85fb70311..3f342195f0 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1630,6 +1630,7 @@ static void fts5SegIterInit( ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; @@ -1645,12 +1646,18 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ int nPos; int bDummy; - i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); - i += nPos; + if( eDetail==FTS5_DETAIL_NONE ){ + /* todo */ + + }else{ + i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); + i += nPos; + } if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); pIter->iRowid += iDelta; + /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); @@ -1754,8 +1761,10 @@ static void fts5SegIterNext( pIter->iRowidOffset--; pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; - iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); - iOff += nPos; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); + iOff += nPos; + } fts5GetVarint(&a[iOff], (u64*)&iDelta); pIter->iRowid -= iDelta; fts5SegIterLoadNPos(p, pIter); @@ -1893,6 +1902,7 @@ static void fts5SegIterNext( ** the doclist. */ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ + int eDetail = p->pConfig->eDetail; Fts5DlidxIter *pDlidx = pIter->pDlidx; Fts5Data *pLast = 0; int pgnoLast = 0; @@ -1907,7 +1917,9 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ - pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); + if( eDetail!=FTS5_DETAIL_NONE ){ + pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); + } /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to @@ -2895,6 +2907,9 @@ static void fts5ChunkIterate( int pgno = pSeg->iLeafPgno; int pgnoSave = 0; + /* This function does notmwork with detail=none databases. */ + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } @@ -3318,8 +3333,7 @@ static void fts5WriteAppendTerm( static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, - i64 iRowid, - int nPos + i64 iRowid ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; @@ -3346,8 +3360,6 @@ static void fts5WriteAppendRowid( pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; - - fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); } } @@ -3543,6 +3555,7 @@ static void fts5IndexMergeLevel( Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ + int eDetail = p->pConfig->eDetail; assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); @@ -3612,11 +3625,21 @@ static void fts5IndexMergeLevel( /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ - nPos = pSegIter->nPos*2 + pSegIter->bDel; - fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos); + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); - /* Append the position-list data to the output */ - fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + if( eDetail==FTS5_DETAIL_NONE ){ + if( pSegIter->bDel ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + if( pSegIter->nPos>0 ){ + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); + } + } + }else{ + /* Append the position-list data to the output */ + nPos = pSegIter->nPos*2 + pSegIter->bDel; + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); + } } /* Flush the last leaf page to disk. Set the output segment b-tree height @@ -4382,7 +4405,7 @@ static void fts5MergeRowidLists( if( p->rc ) return; fts5NextRowid(p1, &i1, &iRowid1); - fts5NextRowid(p1, &i2, &iRowid2); + fts5NextRowid(p2, &i2, &iRowid2); while( i1>=0 || i2>=0 ){ if( i1>=0 && (i2<0 || iRowid1 Date: Thu, 31 Dec 2015 18:39:14 +0000 Subject: [PATCH 09/44] Fix the fts5 integrity-check code so that it works with detail=none tables. FossilOrigin-Name: 3a1df69e58e5830da1dff158eedbe3817743f58f --- ext/fts5/fts5_index.c | 45 +++++++++++++++++---------- ext/fts5/fts5_storage.c | 33 +++++++++++++++++--- ext/fts5/test/fts5simple2.test | 56 ++++++++++++++++++++++++---------- manifest | 16 +++++----- manifest.uuid | 2 +- 5 files changed, 106 insertions(+), 46 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 3f342195f0..ef27d73c31 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5199,6 +5199,7 @@ static int fts5QueryCksum( int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ + int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; Fts5Buffer buf = {0, 0, 0}; @@ -5206,17 +5207,24 @@ static int fts5QueryCksum( while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); - if( rc==SQLITE_OK ){ - Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); - sReader.bEof==0; - sqlite3Fts5PoslistReaderNext(&sReader) - ){ - int iCol = FTS5_POS2COLUMN(sReader.iPos); - int iOff = FTS5_POS2OFFSET(sReader.iPos); - cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + + if( eDetail==FTS5_DETAIL_NONE ){ + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); + }else{ + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); + if( rc==SQLITE_OK ){ + Fts5PoslistReader sReader; + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); + sReader.bEof==0; + sqlite3Fts5PoslistReaderNext(&sReader) + ){ + int iCol = FTS5_POS2COLUMN(sReader.iPos); + int iOff = FTS5_POS2OFFSET(sReader.iPos); + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); + } } + } + if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNext(pIdxIter); } } @@ -5523,6 +5531,7 @@ static void fts5IndexIntegrityCheckSegment( ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ + int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5IndexIter *pIter; /* Used to iterate through entire index */ @@ -5574,12 +5583,16 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); - poslist.n = 0; - fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist); - while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ - int iCol = FTS5_POS2COLUMN(iPos); - int iTokOff = FTS5_POS2OFFSET(iPos); - cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + if( eDetail==FTS5_DETAIL_NONE ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + }else{ + poslist.n = 0; + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index fe3ce829a1..f047fd99dc 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -853,8 +853,23 @@ static int fts5StorageIntegrityCallback( pCtx->szCol++; } - iPos = pTermset ? pCtx->iCol : pCtx->szCol-1; - iCol = pTermset ? 0 : pCtx->iCol; + switch( pCtx->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + iPos = pCtx->szCol-1; + iCol = pCtx->iCol; + break; + + case FTS5_DETAIL_COLUMNS: + iPos = pCtx->iCol; + iCol = 0; + break; + + default: + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); + iPos = 0; + iCol = 0; + break; + } rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); if( rc==SQLITE_OK && bPresent==0 ){ @@ -912,11 +927,14 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ @@ -932,9 +950,14 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + if( rc!=SQLITE_OK ) break; } rc2 = sqlite3_reset(pScan); diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index 259aff0953..f99ea6112e 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -109,35 +109,59 @@ do_execsql_test 7.0 { INSERT INTO t1 VALUES('a1 b2'); COMMIT; } -do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} -do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} +do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} +do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} #------------------------------------------------------------------------- # reset_db -do_execsql_test 7.0 { +do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a1 b1 c1'); INSERT INTO t1 VALUES('a2 b2 c2'); INSERT INTO t1 VALUES('a1 b1 c1'); } -do_execsql_test 7.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} -do_execsql_test 7.0.2 { SELECT rowid FROM t1('a1') } {1 3} -do_execsql_test 7.0.3 { SELECT rowid FROM t1('c2') } {2} +do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2} -do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} -do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} -do_execsql_test 7.0.7 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} +do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} +do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} +do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} -do_execsql_test 7.1.0 { INSERT INTO t1(t1) VALUES('optimize') } +do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') } -do_execsql_test 7.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} -do_execsql_test 7.1.2 { SELECT rowid FROM t1('a1') } {1 3} -do_execsql_test 7.1.3 { SELECT rowid FROM t1('c2') } {2} +do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} +do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3} +do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2} -do_execsql_test 7.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} -do_execsql_test 7.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} -do_execsql_test 7.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} +do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} +do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} +do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 9.0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1'); + INSERT INTO t1 VALUES('a2 b2 c2'); + INSERT INTO t1 VALUES('a1 b1 c1'); +} +do_execsql_test 9.0.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +reset_db +do_execsql_test 9.1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + INSERT INTO t1 VALUES('a1 b1 c1', 'x y z'); + INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3'); + INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z'); +} +do_execsql_test 9.2.1 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} finish_test diff --git a/manifest b/manifest index dc56eb5d2c..c173c4014d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\sfts5\sdetail=none\stables.\sSome\sstill\sremain. -D 2015-12-31T17:36:58.906 +C Fix\sthe\sfts5\sintegrity-check\scode\sso\sthat\sit\sworks\swith\sdetail=none\stables. +D 2015-12-31T18:39:14.423 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -103,9 +103,9 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 40eac312bdcb67552fe6dc2e0a4bad67d9273e3d +F ext/fts5/fts5_index.c a7cfe8e88621b473d749268f918b54b81a067c2e F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89 -F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179 +F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e @@ -170,7 +170,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 -F ext/fts5/test/fts5simple2.test 9b1e4c44fc63bd0cbb3ea639cd0ad9fa764849bf +F ext/fts5/test/fts5simple2.test a1a9f9c443f8852f412ee9658ef667086ccb74d7 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ac8f4cf0cede6bcbe47eeefb85d80a27e9278212 -R 4d80cd94292f49ff63b2b342f7b40c82 +P 6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca +R 70fdbbf97fa363362bfe1572d2790ec2 U dan -Z 84b26c5644e7f6dfb95a54fd72f26ecc +Z 9907fb7300f9e53b05e209bde8721940 diff --git a/manifest.uuid b/manifest.uuid index bcbd354a20..7093688530 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca \ No newline at end of file +3a1df69e58e5830da1dff158eedbe3817743f58f \ No newline at end of file From fe0c3cfee154f8f5f24c10535507f4d8885a3c5c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Jan 2016 19:01:56 +0000 Subject: [PATCH 10/44] Changes to run many fts5 tests with detail=none and detail=col tables as well as the default detail=full. Also fixes for the bugs uncovered by running said tests. FossilOrigin-Name: 6322a1d984e7946735bace8a069ef24b31754b3b --- ext/fts5/fts5Int.h | 1 + ext/fts5/fts5_expr.c | 62 +++++++++++++ ext/fts5/fts5_index.c | 128 +++++++++++++++++---------- ext/fts5/fts5_main.c | 76 +++++++++++----- ext/fts5/test/fts5_common.tcl | 27 ++++++ ext/fts5/test/fts5aa.test | 60 +++++++------ ext/fts5/test/fts5ab.test | 28 +++--- ext/fts5/test/fts5ac.test | 154 +++++++++++++-------------------- ext/fts5/test/fts5ad.test | 32 +++---- ext/fts5/test/fts5ae.test | 52 ++++++----- ext/fts5/test/fts5af.test | 56 ++++++------ ext/fts5/test/fts5detail.test | 31 ++++++- ext/fts5/test/fts5simple2.test | 106 +++++++++++++++++++++++ manifest | 36 ++++---- manifest.uuid | 2 +- 15 files changed, 567 insertions(+), 284 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4ef99ebae1..c369687042 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -646,6 +646,7 @@ Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int ); +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ebd767a617..0a74c7884a 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2327,6 +2327,68 @@ int sqlite3Fts5ExprPopulatePoslists( ); } +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + if( pNode ){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + return 1; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + if( bRet==0 ){ + fts5ExprClearPoslists(pNode); + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + return 1; + } + } + } +} +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + /* ** This function is only called for detail=columns tables. */ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ef27d73c31..49d8638f33 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1494,12 +1494,13 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ int iOff = pIter->iLeafOffset; /* Offset to read at */ ASSERT_SZLEAF_OK(pIter->pLeaf); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); pIter->bDel = 0; pIter->nPos = 1; - if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + if( iOffpLeaf->p[iOff]==0 ){ pIter->bDel = 1; iOff++; - if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + if( iOffpLeaf->p[iOff]==0 ){ pIter->nPos = 1; iOff++; }else{ @@ -1643,13 +1644,16 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; - int nPos; - int bDummy; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ - + if( iiEndofDoclist = iOff; bNewTerm = 1; } - if( iOff>=pLeaf->szLeaf ){ + assert_nc( iOffszLeaf + || p->pConfig->eDetail==FTS5_DETAIL_NONE + ); + if( iOff>pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } @@ -1896,6 +1903,11 @@ static void fts5SegIterNext( #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOffiLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); } +#else + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + assert( eDetail==FTS5_DETAIL_NONE || iPoslist==( + pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) + )); + pIter->iLeafOffset = iPoslist; +#endif /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to @@ -2003,11 +2029,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#define fts5IndexSkipVarint(a, iOff) { \ - int iEnd = iOff+9; \ - while( (a[iOff++] & 0x80) && iOffpConfig->pgsz; - + int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -3870,12 +3891,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffn<=pBuf->nSpace ); - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); - }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffp[pBuf->n++] = 0; + iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - if( iPos>=nCopy ) break; } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; } - iOff += nCopy; } } @@ -4408,16 +4442,18 @@ static void fts5MergeRowidLists( fts5NextRowid(p2, &i2, &iRowid2); while( i1>=0 || i2>=0 ){ if( i1>=0 && (i2<0 || iRowid1iOut ); fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); iOut = iRowid1; fts5NextRowid(p1, &i1, &iRowid1); }else{ + assert( iOut==0 || iRowid2>iOut ); fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); iOut = iRowid2; - fts5NextRowid(p2, &i2, &iRowid2); if( i1>=0 && iRowid1==iRowid2 ){ fts5NextRowid(p1, &i1, &iRowid1); } + fts5NextRowid(p2, &i2, &iRowid2); } } @@ -5217,7 +5253,7 @@ static int fts5QueryCksum( for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) - ){ + ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); @@ -5584,7 +5620,9 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); if( eDetail==FTS5_DETAIL_NONE ){ - cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } }else{ poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index cd43308513..95d72ebbc1 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -730,15 +730,18 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); - for(i=0; i<(pSorter->nIdx-1); i++){ - int iVal; - a += fts5GetVarint32(a, iVal); - iOff += iVal; - pSorter->aIdx[i] = iOff; + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; } - pSorter->aIdx[i] = &aBlob[nBlob] - a; - pSorter->aPoslist = a; fts5CsrNewrow(pCsr); } @@ -1678,15 +1681,15 @@ static int fts5ApiColumnText( } static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int n; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistWriter *aWriter; int i; -assert( pCsr->pSorter==0 ); /* fixme */ aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); if( aWriter==0 ) rc = SQLITE_NOMEM; for(i=0; inCol && rc==SQLITE_OK; i++){ @@ -1699,11 +1702,14 @@ assert( pCsr->pSorter==0 ); /* fixme */ } } sqlite3_free(aWriter); + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); n = pSorter->aIdx[iPhrase] - i1; @@ -2204,20 +2210,46 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: - /* Append the varints */ - for(i=0; i<(nPhrase-1); i++){ - const u8 *dummy; - int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); - sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); - } + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } - /* Append the position lists */ - for(i=0; ipExpr, i, &pPoslist); - sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + /* Append the position lists */ + for(i=0; ipExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && ipExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 64af5c59b3..3bcd33dcfc 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -365,3 +365,30 @@ proc fts5_tokenize_split {text} { set ret } +#------------------------------------------------------------------------- +# +proc foreach_detail_mode {prefix script} { + set saved $::testprefix + foreach d [list full col none] { + set s [string map [list %DETAIL% $d] $script] + set ::detail $d + set ::testprefix "$prefix-$d" + reset_db + uplevel $s + unset ::detail + } + set ::testprefix $saved +} + +proc detail_check {} { + if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { + error "not in foreach_detail_mode {...} block" + } +} +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } +proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } + + + + diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 59b8ab86d1..e00bcfcdcc 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -21,6 +21,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; @@ -41,9 +43,9 @@ do_execsql_test 1.1 { #------------------------------------------------------------------------- # -reset_db + do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); @@ -66,11 +68,12 @@ do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } + #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } foreach {i x y} { 1 {g f d b f} {h h e i a} @@ -93,7 +96,7 @@ foreach {i x y} { # reset_db do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -117,7 +120,7 @@ foreach {i x y} { # reset_db do_execsql_test 5.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -141,7 +144,7 @@ foreach {i x y} { # reset_db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -276,7 +279,7 @@ for {set i 1} {$i <= 10} {incr i} { # reset_db do_execsql_test 10.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } set d10 { 1 {g f d b f} {h h e i a} @@ -309,19 +312,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { - CREATE VIRTUAL TABLE rank USING fts5(a, b, c); + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { - CREATE VIRTUAL TABLE t2 USING fts5(x,y); + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); } {} do_catchsql_test 12.2 { @@ -337,7 +340,7 @@ do_test 12.3 { # reset_db do_execsql_test 13.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} @@ -361,7 +364,7 @@ do_execsql_test 13.6 { # reset_db do_execsql_test 14.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' @@ -371,6 +374,10 @@ do_execsql_test 14.1 { INSERT INTO t1 SELECT * FROM d LIMIT 200; } +do_execsql_test 15.x { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + do_test 14.2 { set nRow 0 db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { @@ -433,7 +440,7 @@ do_catchsql_test 16.2 { # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE b2 USING fts5(x); + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); @@ -447,18 +454,20 @@ do_test 17.2 { set res } {{a b c} {a b c} {a b c}} -reset_db -do_execsql_test 18.1 { - CREATE VIRTUAL TABLE c2 USING fts5(x, y); - INSERT INTO c2 VALUES('x x x', 'x x x'); - SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; -} {1} +if {[string match n* %DETAIL%]==0} { + reset_db + do_execsql_test 17.3 { + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO c2 VALUES('x x x', 'x x x'); + SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; + } {1} +} #------------------------------------------------------------------------- # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE uio USING fts5(ttt); + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; @@ -505,8 +514,8 @@ do_execsql_test 17.9 { #-------------------------------------------------------------------- # do_execsql_test 18.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); - CREATE VIRTUAL TABLE t2 USING fts5(c, d); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } @@ -522,7 +531,7 @@ do_execsql_test 18.3 { # reset_db do_execsql_test 19.0 { - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; @@ -533,7 +542,7 @@ do_execsql_test 19.0 { # reset_db do_execsql_test 20.0 { - CREATE VIRTUAL TABLE temp.tmp USING fts5(x); + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] @@ -545,6 +554,7 @@ do_test 20.1 { execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } } $::ids +} finish_test diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 0746e64326..95da2cd2eb 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('hello', 'world'); INSERT INTO t1 VALUES('one two', 'three four'); INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); @@ -57,7 +59,7 @@ do_execsql_test 1.6 { reset_db do_execsql_test 2.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); INSERT INTO t1 VALUES('one'); INSERT INTO t1 VALUES('two'); @@ -159,7 +161,7 @@ foreach {tn expr res} { # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE s1 USING fts5(x); + CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); } foreach {tn doc} [list \ 1 [string repeat {a x } 1500000] \ @@ -172,8 +174,14 @@ do_execsql_test 4.3 { SELECT rowid FROM s1 WHERE s1 MATCH 'x' } {1 2} -do_execsql_test 4.4 { - SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' +if {[detail_is_full]} { + do_execsql_test 4.4 { + SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' + } {1 2} +} + +do_execsql_test 4.5 { + SELECT rowid FROM s1 WHERE s1 MATCH 'a x' } {1 2} #------------------------------------------------------------------------- @@ -182,7 +190,7 @@ do_execsql_test 4.4 { # (L-2) is larger than it. # do_execsql_test 5.0 { - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -222,7 +230,7 @@ do_test 5.2 { do_test 5.3 { execsql { DROP TABLE s2; - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -241,7 +249,7 @@ do_test 5.4 { #------------------------------------------------------------------------- # do_execsql_test 6.0 { - CREATE VIRTUAL TABLE s3 USING fts5(x); + CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); BEGIN; INSERT INTO s3 VALUES('a b c'); INSERT INTO s3 VALUES('A B C'); @@ -276,13 +284,13 @@ do_test 6.4 { #------------------------------------------------------------------------- # set doc [string repeat "a b c " 500] -breakpoint do_execsql_test 7.0 { - CREATE VIRTUAL TABLE x1 USING fts5(x); + CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); INSERT INTO x1 VALUES($doc); } +} ;# foreach_detail_mode... finish_test diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index ff2affcce1..c3163147cb 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} @@ -205,8 +207,6 @@ proc collist_data {expr} { # End of test code #------------------------------------------------------------------------- -if 0 { - foreach {tn2 sql} { 1 {} 2 {BEGIN} @@ -215,7 +215,7 @@ foreach {tn2 sql} { fts5_aux_test_functions db do_execsql_test 1.$tn2.0 { - CREATE VIRTUAL TABLE xx USING fts5(x,y); + CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } @@ -228,31 +228,55 @@ foreach {tn2 sql} { execsql { INSERT INTO xx(xx) VALUES('integrity-check') } } {} - #------------------------------------------------------------------------- - # Test phrase queries. + # The following work with all detail= modes. # - foreach {tn phrase} { - 1 "o" - 2 "b q" - 3 "e a e" - 4 "m d g q q b k b w f q q p p" - 5 "l o o l v v k" - 6 "a" - 7 "b" - 8 "c" - 9 "no" - 10 "L O O L V V K" + foreach {tn expr} { + 1 "a AND b" + 2 "a OR b" + 3 "o" + 4 "b q" + 5 "e a e" + 6 "m d g q q b k b w f q q p p" + 7 "l o o l v v k" + 8 "a" + 9 "b" + 10 "c" + 11 "no" + 12 "L O O L V V K" + 13 "a AND b AND c" } { - set expr "\"$phrase\"" set res [poslist_data 1 $expr] - - do_execsql_test 1.$tn2.1.2.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.1.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.1.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.1.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res + } + + if {[detail_is_none] || [detail_is_col]} continue + + #------------------------------------------------------------------------- + # Test phrase queries. + # + foreach {tn expr} { + 1 "b + q" + 2 "e + a + e" + 3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" + 4 "l + o + o + l + v + v + k" + 5 "L + O + O + L + V + V + K" + } { + set res [poslist_data 1 $expr] + + do_execsql_test 1.$tn2.2.$tn.p.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -261,25 +285,21 @@ foreach {tn2 sql} { # Test some AND and OR queries. # foreach {tn expr} { - 1.1 "a AND b" - 1.2 "a+b AND c" - 1.3 "d+c AND u" - 1.4 "d+c AND u+d" + 1.1 "a+b AND c" + 1.2 "d+c AND u" + 1.3 "d+c AND u+d" - 2.1 "a OR b" - 2.2 "a+b OR c" - 2.3 "d+c OR u" - 2.4 "d+c OR u+d" - - 3.1 { a AND b AND c } + 2.1 "a+b OR c" + 2.2 "d+c OR u" + 2.3 "d+c OR u+d" } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -308,12 +328,12 @@ foreach {tn2 sql} { 4.4 {{"y" y}:b} } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.3.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -334,12 +354,12 @@ foreach {tn2 sql} { 9 { y : NEAR(r c) } } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.4.1.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.5.1.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.4.1.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.5.1.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -350,14 +370,14 @@ foreach {tn2 sql} { foreach {tn expr tclexpr} { 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} } { - do_execsql_test 1.$tn2.5.$tn { + do_execsql_test 1.$tn2.6.$tn { SELECT fts5_expr_tcl($expr, 'N $x') } [list $tclexpr] } #------------------------------------------------------------------------- # - do_execsql_test 1.$tn2.6.integrity { + do_execsql_test 1.$tn2.7.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} @@ -389,68 +409,18 @@ foreach {tn2 sql} { 19 { c NOT b OR a AND d } } { set res [poslist_data 0 $expr $bAsc] - do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res + do_execsql_test 1.$tn2.7.$bAsc.$tn.[llength $res] $sql $res } } } +} + do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} -} -#set data [lrange $data 0 5] - -# Some tests for detail=col tables and detail=none. -# -foreach {tn2 sql} { - 1 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); - } - 2 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); - BEGIN; - } - 3 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); - BEGIN; - } - 4 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); - } -} { - reset_db - fts5_aux_test_functions db - - execsql $sql - - do_execsql_test 3.$tn2.0 { - INSERT INTO xx(xx, rank) VALUES('pgsz', 32); - } - - do_test 3.$tn2.1.1 { - foreach {id x y} $data { - execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } - } - execsql { INSERT INTO xx(xx) VALUES('integrity-check') } - } {} - - foreach {tn q} { - 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" - 11 "o a" 12 "c AND d" 13 "o OR a" 14 "c OR d" - } { - set res [poslist_data 1 $q] - do_execsql_test 3.$tn2.1.2.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q - } $res - - set res [collist_data $q] - do_execsql_test 3.$tn2.1.2.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $q - } $res - } - -} +finish_test finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index cc86a6b2eb..b875354534 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -22,8 +22,12 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + +if {[detail_is_none]==0} continue + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE yy USING fts5(x, y); + CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); @@ -53,39 +57,23 @@ foreach {tn match res} { foreach {T create} { 2 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 3 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 4 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } 5 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - BEGIN; - } - - 6 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - } - - 7 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - } - - 8 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } @@ -251,5 +239,7 @@ foreach {T create} { catchsql COMMIT } +} + finish_test diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index ded73d472f..5153306d19 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -55,7 +57,7 @@ fts5_aux_test_functions db #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(x, y); + CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); } @@ -76,31 +78,35 @@ do_execsql_test 2.2 { 2 {1.0.2 1.0.10} } -do_execsql_test 2.3 { - SELECT rowid, fts5_test_poslist(t2) FROM t2 - WHERE t2 MATCH 'y:o' ORDER BY rowid; -} { - 1 {0.1.3 0.1.7} +if {[detail_is_full]} { + do_execsql_test 2.3 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'y:o' ORDER BY rowid; + } { + 1 {0.1.3 0.1.7} + } } #------------------------------------------------------------------------- # do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t3 USING fts5(x, y); + CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); INSERT INTO t3 VALUES( 'r c', ''); } -do_execsql_test 3.1 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; -} { - 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} -} +if {[detail_is_full]} { + do_execsql_test 3.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; + } { + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} + } -do_execsql_test 3.2 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; -} { - 2 {0.0.0 1.0.1} + do_execsql_test 3.2 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; + } { + 2 {0.0.0 1.0.1} + } } do_execsql_test 3.3 { @@ -116,7 +122,7 @@ do_execsql_test 3.3 { #------------------------------------------------------------------------- # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t4 USING fts5(x, y); + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); } @@ -134,7 +140,7 @@ reset_db fts5_aux_test_functions db do_execsql_test 5.1 { - CREATE VIRTUAL TABLE t5 USING fts5(x, y); + CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); INSERT INTO t5 VALUES('', 'a'); INSERT INTO t5 VALUES('a', ''); @@ -182,7 +188,7 @@ do_execsql_test 5.5 { reset_db fts5_aux_test_functions db do_execsql_test 6.1 { - CREATE VIRTUAL TABLE t6 USING fts5(x, y); + CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } @@ -200,7 +206,7 @@ do_execsql_test 6.2 { reset_db fts5_aux_test_functions db do_execsql_test 7.1 { - CREATE VIRTUAL TABLE t7 USING fts5(x, y); + CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); } do_test 7.2 { foreach {x y} { @@ -240,7 +246,7 @@ do_execsql_test 7.4 { #------------------------------------------------------------------------- # do_test 8.1 { - execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) } + execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } foreach {rowid x y} { 0 {A o} {o o o C o o o o o o o o} 1 {o o B} {o o o C C o o o o o o o} @@ -300,5 +306,7 @@ foreach {tn q cnt} { } $cnt } +} + finish_test diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index 8c50f84866..d6b2241568 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -24,9 +24,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } proc do_snippet_test {tn doc match res} { @@ -111,34 +112,37 @@ foreach {tn doc res} { do_snippet_test 1.$tn $doc X $res } -foreach {tn doc res} { - 1.1 {X Y o o o o o} {[X Y] o o o o o} - 1.2 {o X Y o o o o} {o [X Y] o o o o} - 1.3 {o o X Y o o o} {o o [X Y] o o o} - 1.4 {o o o X Y o o} {o o o [X Y] o o} - 1.5 {o o o o X Y o} {o o o o [X Y] o} - 1.6 {o o o o o X Y} {o o o o o [X Y]} +if {[detail_is_full]} { + foreach {tn doc res} { + 1.1 {X Y o o o o o} {[X Y] o o o o o} + 1.2 {o X Y o o o o} {o [X Y] o o o o} + 1.3 {o o X Y o o o} {o o [X Y] o o o} + 1.4 {o o o X Y o o} {o o o [X Y] o o} + 1.5 {o o o o X Y o} {o o o o [X Y] o} + 1.6 {o o o o o X Y} {o o o o o [X Y]} - 2.1 {X Y o o o o o o} {[X Y] o o o o o...} - 2.2 {o X Y o o o o o} {o [X Y] o o o o...} - 2.3 {o o X Y o o o o} {o o [X Y] o o o...} - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} + 2.1 {X Y o o o o o o} {[X Y] o o o o o...} + 2.2 {o X Y o o o o o} {o [X Y] o o o o...} + 2.3 {o o X Y o o o o} {o o [X Y] o o o...} + 2.4 {o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o X Y} {...o o o o o [X Y]} - 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} - 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} - 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} - -} { - do_snippet_test 2.$tn $doc "X + Y" $res + 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} + 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} + 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} + 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} + } { + do_snippet_test 2.$tn $doc "X + Y" $res + } } +} ;# foreach_detail_mode + finish_test diff --git a/ext/fts5/test/fts5detail.test b/ext/fts5/test/fts5detail.test index 59f4b0e3c3..5bdd14424e 100644 --- a/ext/fts5/test/fts5detail.test +++ b/ext/fts5/test/fts5detail.test @@ -169,8 +169,8 @@ proc matchdata {expr {bAsc 1}} { foreach {tn tbl} { 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } + 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } } { -#break reset_db fts5_aux_test_functions db execsql $tbl @@ -198,7 +198,6 @@ foreach {tn tbl} { #------------------------------------------------------------------------- # Simple tests for detail=none tables. # -breakpoint do_execsql_test 4.0 { CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); @@ -209,5 +208,33 @@ do_catchsql_test 4.1 { SELECT * FROM t4('a:a') } {1 {fts5: column queries are not supported (detail=none)}} +#------------------------------------------------------------------------- +# Test that for the same content detail=none uses less space than +# detail=col, and that detail=col uses less space than detail=full +# +reset_db +do_test 5.1 { + foreach {tbl detail} {t1 none t2 col t3 full} { + execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" + foreach {rowid x y z} $::data { + execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" + } + } +} {} + +do_execsql_test 5.2 { + SELECT + (SELECT sum(length(block)) from t1_data) < + (SELECT sum(length(block)) from t2_data) +} {1} + +do_execsql_test 5.3 { + SELECT + (SELECT sum(length(block)) from t2_data) < + (SELECT sum(length(block)) from t3_data) +} {1} + + + finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index f99ea6112e..bb76250e9f 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -19,6 +19,8 @@ ifcapable !fts5 { return } +if 1 { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a b c'); @@ -163,5 +165,109 @@ do_execsql_test 9.2.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('b1'); + INSERT INTO t1 VALUES('b1'); + DELETE FROM t1 WHERE rowid=1; +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1('b1'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t1('b1') ORDER BY rowid DESC; +} {2} + +do_execsql_test 10.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH d(x,y) AS ( + SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d + ) + INSERT INTO t1 SELECT * FROM d LIMIT 23; +} + +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } +do_execsql_test 11.2 { + SELECT rowid FROM t1; +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.3 { + SELECT rowid FROM t1('xyz'); +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); + INSERT INTO yy VALUES('in if'); + INSERT INTO yy VALUES('if'); +} {} + +do_execsql_test 12.1 { + SELECT rowid FROM yy('i*'); +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); +} {} +foreach {rowid a} { + 0 {f} + 1 {u} + 2 {k} + 3 {a} + 4 {a} + 5 {u} + 6 {u} + 7 {u} + 8 {f} + 9 {f} + 10 {a} + 11 {p} + 12 {f} + 13 {u} + 14 {a} + 15 {a} +} { + do_execsql_test 13.1.$rowid { + INSERT INTO t1(rowid, a) VALUES($rowid, $a); + } +} + +} + +#------------------------------------------------------------------------- +# +reset_db +fts5_aux_test_functions db +do_execsql_test 14.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c d'); +} {} + +do_execsql_test 14.1 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {0.0.1} + finish_test diff --git a/manifest b/manifest index c173c4014d..ce2b5eb30f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sfts5\sintegrity-check\scode\sso\sthat\sit\sworks\swith\sdetail=none\stables. -D 2015-12-31T18:39:14.423 +C Changes\sto\srun\smany\sfts5\stests\swith\sdetail=none\sand\sdetail=col\stables\sas\swell\sas\sthe\sdefault\sdetail=full.\sAlso\sfixes\sfor\sthe\sbugs\suncovered\sby\srunning\ssaid\stests. +D 2016-01-02T19:01:56.026 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,14 +97,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 -F ext/fts5/fts5Int.h ab5f6f1f65652310a644d97d12aeabf8ab5ed17e +F ext/fts5/fts5Int.h cff7dd3131ba0db6f699df97237bf993f22c1a1f F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5 +F ext/fts5/fts5_expr.c 898e0bb53280b2c4c2702a2dc08dec731c7b32fe F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c a7cfe8e88621b473d749268f918b54b81a067c2e -F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89 +F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c +F ext/fts5/fts5_main.c f33439bde2e1023888e9b7f16e463e42fe4c00c5 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 @@ -114,13 +114,13 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl f923471d572d1e283b7d6985ce3cda62dae6fd45 -F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 -F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 1d2a10d4c13dcfd326287f51df3b7d90761c212c -F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3 -F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe -F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a +F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824 +F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc +F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b +F ext/fts5/test/fts5ac.test 9a3c5eb052a5e8c871d56d3d0f144e5fdcf075c2 +F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc +F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 +F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 @@ -139,7 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 -F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 +F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 @@ -170,7 +170,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 -F ext/fts5/test/fts5simple2.test a1a9f9c443f8852f412ee9658ef667086ccb74d7 +F ext/fts5/test/fts5simple2.test d453e3ca100e08078a46b12547417891b326240a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca -R 70fdbbf97fa363362bfe1572d2790ec2 +P 3a1df69e58e5830da1dff158eedbe3817743f58f +R 584d5f1c8b7a15ac2eae2ab1ffe71c4a U dan -Z 9907fb7300f9e53b05e209bde8721940 +Z cff9ad15204445a84003e93595871343 diff --git a/manifest.uuid b/manifest.uuid index 7093688530..4a0bbbdcd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a1df69e58e5830da1dff158eedbe3817743f58f \ No newline at end of file +6322a1d984e7946735bace8a069ef24b31754b3b \ No newline at end of file From 5af10196912b7aae7a848fd06b54405d738d128b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Jan 2016 16:19:51 +0000 Subject: [PATCH 11/44] Update more fts5 tests to run in detail=none and detail=column modes as well as the default detail=full. FossilOrigin-Name: 3fcf3b1e24483b2cd1c1710f053ed8403e09106d --- ext/fts5/test/fts5ag.test | 21 +++++--- ext/fts5/test/fts5ah.test | 46 +++++++++++----- ext/fts5/test/fts5ai.test | 5 +- ext/fts5/test/fts5ak.test | 107 ++++++++++++++++++++------------------ ext/fts5/test/fts5al.test | 59 ++++++++++++--------- manifest | 20 +++---- manifest.uuid | 2 +- 7 files changed, 152 insertions(+), 108 deletions(-) diff --git a/ext/fts5/test/fts5ag.test b/ext/fts5/test/fts5ag.test index 42a588f56c..de126a25f0 100644 --- a/ext/fts5/test/fts5ag.test +++ b/ext/fts5/test/fts5ag.test @@ -33,8 +33,10 @@ ifcapable !fts5 { # ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] # +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, z); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%); } do_test 1.1 { @@ -119,19 +121,24 @@ foreach {tn expr} { 2.3 c 2.4 d - 2.5 {"m m"} - 2.6 {e + s} - 3.0 {a AND b} 3.1 {a OR b} 3.2 {b OR c AND d} - 3.3 {NEAR(c d)} } { do_fts5ag_test $tn $expr - - if {[set_test_counter errors]} break } +if {[detail_is_full]} { + foreach {tn expr} { + 4.1 {"m m"} + 4.2 {e + s} + 4.3 {NEAR(c d)} + } { + do_fts5ag_test $tn $expr + } +} + +} ;# foreach_detail_mode finish_test diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 6d7e39f793..b7beb5655b 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -21,12 +21,16 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + #------------------------------------------------------------------------- # This file contains tests for very large doclists. # +set Y [list] +set W [list] do_test 1.0 { - execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) } + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) } execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } set v {w w w w w w w w w w w w w w w w w w w w} execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) } @@ -70,7 +74,12 @@ do_test 1.4 { set nRead [reads] execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } set nReadX [expr [reads] - $nRead] - expr $nReadX>1000 + #puts -nonewline "(nReadX=$nReadX)" + if {[detail_is_full]} { set expect 1000 } + if {[detail_is_col]} { set expect 250 } + if {[detail_is_none]} { set expect 80 } + + expr $nReadX>$expect } {1} do_test 1.5 { @@ -87,17 +96,22 @@ foreach {tn q res} " 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] " { + if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue + + if {[detail_is_full]} { set ratio 8 } + if {[detail_is_col]} { set ratio 4 } + if {[detail_is_none]} { set ratio 2 } do_test 1.6.$tn.1 { set n [execsql_reads $q] #puts -nonewline "(n=$n nReadX=$nReadX)" - expr {$n < ($nReadX / 8)} + expr {$n < ($nReadX / $ratio)} } {1} do_test 1.6.$tn.2 { set n [execsql_reads "$q ORDER BY rowid DESC"] #puts -nonewline "(n=$n nReadX=$nReadX)" - expr {$n < ($nReadX / 8)} + expr {$n < ($nReadX / $ratio)} } {1} do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res] @@ -109,21 +123,26 @@ foreach {tn q res} " # number of pages loaded from disk. # foreach {tn fraction tail cnt} { - 1 0.6 {rowid > 5000} 5000 - 2 0.2 {rowid > 9000} 1000 - 3 0.2 {rowid < 1000} 999 - 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 - 5 0.6 {rowid >= 5000} 5001 - 6 0.2 {rowid >= 9000} 1001 - 7 0.2 {rowid <= 1000} 1000 - 8 0.6 {rowid > '5000'} 5000 - 9 0.2 {rowid > '9000'} 1000 + 1 0.6 {rowid > 5000} 5000 + 2 0.2 {rowid > 9000} 1000 + 3 0.2 {rowid < 1000} 999 + 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 + 5 0.6 {rowid >= 5000} 5001 + 6 0.2 {rowid >= 9000} 1001 + 7 0.2 {rowid <= 1000} 1000 + 8 0.6 {rowid > '5000'} 5000 + 9 0.2 {rowid > '9000'} 1000 10 0.1 {rowid = 444} 1 } { set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" set n [execsql_reads $q] set ret [llength [execsql $q]] + # Because the position lists for 'x' are quite long in this db, the + # advantage is a bit smaller in detail=none mode. Update $fraction to + # reflect this. + if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] } + do_test "1.7.$tn.asc.(n=$n ret=$ret)" { expr {$n < ($fraction*$nReadX) && $ret==$cnt} } {1} @@ -143,6 +162,7 @@ do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} +} ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5ai.test b/ext/fts5/test/fts5ai.test index 63c46fd042..e32c806c46 100644 --- a/ext/fts5/test/fts5ai.test +++ b/ext/fts5/test/fts5ai.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a); + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%); } {} do_execsql_test 1.1 { @@ -49,6 +51,7 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } +} finish_test diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index 4eb28324c9..0f699a601f 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.1 { - CREATE VIRTUAL TABLE ft1 USING fts5(x); + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); INSERT INTO ft1 VALUES('i d d a g i b g d d'); INSERT INTO ft1 VALUES('h d b j c c g a c a'); INSERT INTO ft1 VALUES('e j a e f h b f h h'); @@ -35,6 +37,9 @@ do_execsql_test 1.1 { INSERT INTO ft1 VALUES('i c c f a d g h j e'); INSERT INTO ft1 VALUES('i d i g c d c h b f'); INSERT INTO ft1 VALUES('g d a e h a b c f j'); + + CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%); + INSERT INTO ft2 VALUES('a b c d e f g h i j'); } do_execsql_test 1.2 { @@ -49,19 +54,6 @@ do_execsql_test 1.2 { } do_execsql_test 1.3 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; -} { - {[h d] b j c c g a c a} - {j f [h d] g h i b d f} -} - -do_execsql_test 1.4 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; -} { - {i [d d] a g i b g [d d]} -} - -do_execsql_test 1.5 { SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' } { {[e] j a [e] f h b f h h} @@ -72,57 +64,71 @@ do_execsql_test 1.5 { {g d a [e] h a b c f j} } -do_execsql_test 1.6 { - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; -} { - {i [d d] a g i b g [d d]} -} - -do_execsql_test 2.1 { - CREATE VIRTUAL TABLE ft2 USING fts5(x); - INSERT INTO ft2 VALUES('a b c d e f g h i j'); -} - -do_execsql_test 2.2 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' -} {{a [b c d e] f g h i j}} - -do_execsql_test 2.3 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' -} { - {a [b c d] [e f g] h i j} -} - -do_execsql_test 2.4 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' -} { - {a [b c d] e f g h i j} -} - -do_execsql_test 2.5 { - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' -} { - {a [b c d e] f g h i j} -} - -do_execsql_test 2.6.1 { +do_execsql_test 1.4 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' } { {a b c [d] e [f] g h i j} } -do_execsql_test 2.6.2 { +do_execsql_test 1.5 { SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' } { {a b c [d] e [f] g h i j} } +#------------------------------------------------------------------------- +# Tests below this point require detail=full. +#------------------------------------------------------------------------- +if {[detail_is_full]==0} continue + + +do_execsql_test 2.1 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; +} { + {[h d] b j c c g a c a} + {j f [h d] g h i b d f} +} + +do_execsql_test 2.2 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.3 { + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; +} { + {i [d d] a g i b g [d d]} +} + +do_execsql_test 2.4 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' +} {{a [b c d e] f g h i j}} + +do_execsql_test 2.5 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' +} { + {a [b c d] [e f g] h i j} +} + +do_execsql_test 2.6 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' +} { + {a [b c d] e f g h i j} +} + +do_execsql_test 2.7 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' +} { + {a [b c d e] f g h i j} +} + #------------------------------------------------------------------------- # The example from the docs. # do_execsql_test 3.1 { -- Assuming this: - CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%); INSERT INTO ft VALUES('a b c x c d e'); INSERT INTO ft VALUES('a b c c d e'); INSERT INTO ft VALUES('a b c d e'); @@ -138,6 +144,7 @@ do_execsql_test 3.1 { {[a b c d e]} } +} finish_test diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index b8f8c6ebcd..c0dd2117dd 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -23,8 +23,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.1 { - CREATE VIRTUAL TABLE ft1 USING fts5(x); + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); SELECT * FROM ft1_config; } {version 4} @@ -83,7 +85,7 @@ foreach {tn defn} { # do_execsql_test 3.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('q w e r t y'); INSERT INTO t1 VALUES('y t r e w q'); } @@ -122,11 +124,13 @@ do_execsql_test 3.4.1 { {{0 0 5}} } -do_execsql_test 3.4.2 { - SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' -} { - {{1 0 1}} - {{0 0 2} {1 0 4}} +if {[detail_is_full]} { + do_execsql_test 3.4.2 { + SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' + } { + {{1 0 1}} + {{0 0 2} {1 0 4}} + } } proc coltest {cmd} { @@ -149,7 +153,7 @@ do_execsql_test 3.5.1 { # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e'); INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q'); INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t'); @@ -218,24 +222,26 @@ proc rowidplus {cmd ival} { } sqlite3_fts5_create_function db rowidplus rowidplus -do_execsql_test 4.2.1 { - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' -} { - 10 110 -} -do_execsql_test 4.2.2 { - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' -} { - 10 121 -} +if {[detail_is_full]} { + do_execsql_test 4.2.1 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 110 + } + do_execsql_test 4.2.2 { + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' + } { + 10 121 + } -do_execsql_test 4.2.3 { - SELECT rowid, rank FROM t2 - WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' -} { - 10 122 + do_execsql_test 4.2.3 { + SELECT rowid, rank FROM t2 + WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' + } { + 10 122 + } } proc rowidmod {cmd imod} { @@ -243,7 +249,7 @@ proc rowidmod {cmd imod} { } sqlite3_fts5_create_function db rowidmod rowidmod do_execsql_test 4.3.1 { - CREATE VIRTUAL TABLE t3 USING fts5(x); + CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%); INSERT INTO t3 VALUES('a one'); INSERT INTO t3 VALUES('a two'); INSERT INTO t3 VALUES('a three'); @@ -287,6 +293,7 @@ do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} +} ;# foreach_detail_mode finish_test diff --git a/manifest b/manifest index ce2b5eb30f..ff5d797e2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\srun\smany\sfts5\stests\swith\sdetail=none\sand\sdetail=col\stables\sas\swell\sas\sthe\sdefault\sdetail=full.\sAlso\sfixes\sfor\sthe\sbugs\suncovered\sby\srunning\ssaid\stests. -D 2016-01-02T19:01:56.026 +C Update\smore\sfts5\stests\sto\srun\sin\sdetail=none\sand\sdetail=column\smodes\sas\swell\sas\sthe\sdefault\sdetail=full. +D 2016-01-04T16:19:51.421 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -121,12 +121,12 @@ F ext/fts5/test/fts5ac.test 9a3c5eb052a5e8c871d56d3d0f144e5fdcf075c2 F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c -F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 -F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c -F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 +F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071 +F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5 +F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 -F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test a1b7b6393376bc2adc216527a28f5ae5594069df +F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9 +F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test 2a6241673657b340427f521528f7809ddaa02a9e F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3a1df69e58e5830da1dff158eedbe3817743f58f -R 584d5f1c8b7a15ac2eae2ab1ffe71c4a +P 6322a1d984e7946735bace8a069ef24b31754b3b +R 32e8694f89e6ec744ed1fb193a76a532 U dan -Z cff9ad15204445a84003e93595871343 +Z 780f62d5b61ae786fb811bd6c42602f7 diff --git a/manifest.uuid b/manifest.uuid index 4a0bbbdcd0..3f2a04cfe4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6322a1d984e7946735bace8a069ef24b31754b3b \ No newline at end of file +3fcf3b1e24483b2cd1c1710f053ed8403e09106d \ No newline at end of file From 211bd3a20a50305837d48374d1ced88710511d52 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Jan 2016 19:12:00 +0000 Subject: [PATCH 12/44] Fix an OOM handling problem in fts5 detail=none and detail=col modes. Also a bug in the xInst() API when used with detail=col and column filtering expressions. Update the matchinfo() test function so that 'b' is fast with detail=col tables. FossilOrigin-Name: ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9 --- ext/fts5/fts5Int.h | 5 +- ext/fts5/fts5_expr.c | 55 ++++++++++---- ext/fts5/fts5_main.c | 121 +++++++++++++++++-------------- ext/fts5/fts5_test_mi.c | 27 +++++-- ext/fts5/test/fts5ac.test | 72 +++++++++--------- ext/fts5/test/fts5fault8.test | 49 +++++++++++++ ext/fts5/test/fts5matchinfo.test | 80 ++++++++++++-------- manifest | 23 +++--- manifest.uuid | 2 +- 9 files changed, 279 insertions(+), 155 deletions(-) create mode 100644 ext/fts5/test/fts5fault8.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c369687042..b853bb3a43 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -642,9 +642,10 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); -Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*); int sqlite3Fts5ExprPopulatePoslists( - Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 0a74c7884a..ae00a12c82 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2257,28 +2257,44 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ return nRet; } -Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ - int i; - Fts5PoslistWriter *pRet; - for(i=0; inPhrase; i++){ - Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; - assert( pExpr->apExprPhrase[i]->nTerm==1 ); - pBuf->n = 0; - } - pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase); +struct Fts5PoslistPopulator { + Fts5PoslistWriter writer; + int bOk; /* True if ok to populate */ +}; + +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ + Fts5PoslistPopulator *pRet; + pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); if( pRet ){ - memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase); + int i; + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + pBuf->n = 0; + } } return pRet; } struct Fts5ExprCtx { Fts5Expr *pExpr; - Fts5PoslistWriter *aWriter; + Fts5PoslistPopulator *aPopulator; i64 iOff; }; typedef struct Fts5ExprCtx Fts5ExprCtx; +/* +** TODO: Make this more efficient! +*/ +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + static int fts5ExprPopulatePoslistsCb( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ @@ -2294,13 +2310,14 @@ static int fts5ExprPopulatePoslistsCb( if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ Fts5ExprTerm *pTerm; + if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ int nTerm = strlen(pTerm->zTerm); if( (nTerm==nToken || (nTermbPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( - &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); if( rc ) return rc; break; @@ -2313,15 +2330,25 @@ static int fts5ExprPopulatePoslistsCb( int sqlite3Fts5ExprPopulatePoslists( Fts5Config *pConfig, Fts5Expr *pExpr, - Fts5PoslistWriter *aWriter, + Fts5PoslistPopulator *aPopulator, int iCol, const char *z, int n ){ + int i; Fts5ExprCtx sCtx; sCtx.pExpr = pExpr; - sCtx.aWriter = aWriter; + sCtx.aPopulator = aPopulator; sCtx.iOff = (((i64)iCol) << 32) - 1; + for(i=0; inPhrase; i++){ + Fts5Colset *pColset = pExpr->apExprPhrase[i]->pNode->pNear->pColset; + if( pColset && 0==fts5ExprColsetTest(pColset, iCol) ){ + aPopulator[i].bOk = 0; + }else{ + aPopulator[i].bOk = 1; + } + } + return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 95d72ebbc1..99c9eb05db 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1608,7 +1608,7 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ return rc; } -static int fts5CsrPoslist(Fts5Cursor*, int, const u8**); +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; @@ -1680,28 +1680,32 @@ static int fts5ApiColumnText( return rc; } -static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ +static int fts5CsrPoslist( + Fts5Cursor *pCsr, + int iPhrase, + const u8 **pa, + int *pn +){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; - int n; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ - Fts5PoslistWriter *aWriter; + Fts5PoslistPopulator *aPopulator; int i; - aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); - if( aWriter==0 ) rc = SQLITE_NOMEM; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); + if( aPopulator==0 ) rc = SQLITE_NOMEM; for(i=0; inCol && rc==SQLITE_OK; i++){ int n; const char *z; rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( - pConfig, pCsr->pExpr, aWriter, i, z, n + pConfig, pCsr->pExpr, aPopulator, i, z, n ); } } - sqlite3_free(aWriter); + sqlite3_free(aPopulator); if( pCsr->pSorter ){ sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); } @@ -1712,12 +1716,13 @@ static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - n = pSorter->aIdx[iPhrase] - i1; + *pn = pSorter->aIdx[iPhrase] - i1; *pa = &pSorter->aPoslist[i1]; }else{ - n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); } - return n; + + return rc; } /* @@ -1742,43 +1747,46 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int i; /* Initialize all iterators */ - for(i=0; i=pCsr->nInstAlloc ){ - pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; - aInst = (int*)sqlite3_realloc( - pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 - ); - if( aInst ){ - pCsr->aInst = aInst; - }else{ - rc = SQLITE_NOMEM; - break; + nInst++; + if( nInst>=pCsr->nInstAlloc ){ + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc( + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + ); + if( aInst ){ + pCsr->aInst = aInst; + }else{ + rc = SQLITE_NOMEM; + break; + } } - } - aInst = &pCsr->aInst[3 * (nInst-1)]; - aInst[0] = iBest; - aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); - aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + aInst = &pCsr->aInst[3 * (nInst-1)]; + aInst[0] = iBest; + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); + } } pCsr->nInstCount = nInst; @@ -1981,11 +1989,15 @@ static int fts5ApiPhraseFirst( int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); - pIter->b = &pIter->a[n]; - *piCol = 0; - *piOff = 0; - fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + int n; + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + *piCol = 0; + *piOff = 0; + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); + } + return rc; } static void fts5ApiPhraseNextColumn( @@ -2037,14 +2049,17 @@ static int fts5ApiPhraseFirstColumn( fts5ApiPhraseNextColumn(pCtx, pIter, piCol); } }else{ - int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); - pIter->b = &pIter->a[n]; - if( n<=0 ){ - *piCol = -1; - }else if( pIter->a[0]==0x01 ){ - pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); - }else{ - *piCol = 0; + int n; + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); + if( rc==SQLITE_OK ){ + pIter->b = &pIter->a[n]; + if( n<=0 ){ + *piCol = -1; + }else if( pIter->a[0]==0x01 ){ + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); + }else{ + *piCol = 0; + } } } diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index fb4a2102b4..28331773c0 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -211,18 +211,31 @@ static int fts5MatchinfoLocalCb( int rc = SQLITE_OK; switch( f ){ - case 'b': + case 'b': { + int iPhrase; + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; + for(i=0; inPhrase; iPhrase++){ + Fts5PhraseIter iter; + int iCol; + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); + iCol>=0; + pApi->xPhraseNextColumn(pFts, &iter, &iCol) + ){ + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); + } + } + + break; + } + case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; - if( f=='b' ){ - int nInt = ((p->nCol + 31) / 32) * p->nPhrase; - for(i=0; inCol*p->nPhrase); i++) aOut[i*nMul] = 0; - } + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; for(iPhrase=0; iPhrasenPhrase; iPhrase++){ Fts5PhraseIter iter; diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index c3163147cb..692929167b 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -257,7 +257,43 @@ foreach {tn2 sql} { } $res } - if {[detail_is_none] || [detail_is_col]} continue + if {[detail_is_none]} continue + + #------------------------------------------------------------------------- + # Queries on a specific column. + # + foreach {tn expr} { + 1.1 "x:a" + 1.2 "y:a" + 1.3 "x:b" + 1.4 "y:b" + 2.1 "{x}:a" + 2.2 "{y}:a" + 2.3 "{x}:b" + 2.4 "{y}:b" + + 3.1 "{x y}:a" + 3.2 "{y x}:a" + 3.3 "{x x}:b" + 3.4 "{y y}:b" + + 4.1 {{"x" "y"}:a} + 4.2 {{"y" x}:a} + 4.3 {{x "x"}:b} + 4.4 {{"y" y}:b} + } { + set res [poslist_data 1 $expr] + do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res + } + + if {[detail_is_col]} continue #------------------------------------------------------------------------- # Test phrase queries. @@ -304,40 +340,6 @@ foreach {tn2 sql} { } $res } - #------------------------------------------------------------------------- - # Queries on a specific column. - # - foreach {tn expr} { - 1.1 "x:a" - 1.2 "y:a" - 1.3 "x:b" - 1.4 "y:b" - 2.1 "{x}:a" - 2.2 "{y}:a" - 2.3 "{x}:b" - 2.4 "{y}:b" - - 3.1 "{x y}:a" - 3.2 "{y x}:a" - 3.3 "{x x}:b" - 3.4 "{y y}:b" - - 4.1 {{"x" "y"}:a} - 4.2 {{"y" x}:a} - 4.3 {{x "x"}:b} - 4.4 {{"y" y}:b} - } { - set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - #------------------------------------------------------------------------- # Some NEAR queries. # diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test new file mode 100644 index 0000000000..01a1876955 --- /dev/null +++ b/ext/fts5/test/fts5fault8.test @@ -0,0 +1,49 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault8 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + +if {[detail_is_none]==0} continue + +fts5_aux_test_functions db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); + INSERT INTO t1 VALUES('a b a b', NULL); + INSERT INTO t1 VALUES(NULL, '1 2 1 2'); +} + +do_faultsim_test 1 -faults oom-t* -body { + execsql { + SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' + } +} -test { + faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ + {1 SQLITE_NOMEM} +} + +} + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 21f9b003e7..06f4550b47 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -16,6 +16,8 @@ set testprefix fts5matchinfo # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test ; return } +foreach_detail_mode $testprefix { + proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -27,7 +29,7 @@ db func mit mit sqlite3_fts5_register_matchinfo db do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(content); + CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%); } do_execsql_test 1.1 { @@ -41,7 +43,7 @@ do_execsql_test 1.1 { # Now create an FTS4 table that does not specify matchinfo=fts3. # do_execsql_test 1.2 { - CREATE VIRTUAL TABLE t2 USING fts5(content); + CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%); INSERT INTO t2 SELECT * FROM t1; SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; } {{1 1 1 2 2} {1 1 1 2 2}} @@ -149,9 +151,17 @@ proc normalize2 {list_of_lists} { return $res } +# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5 +# mode is not detail=full. +# +proc do_matchinfo_p_test {tn tbl expr results} { + if {[detail_is_full]} { + uplevel [list do_matchinfo_test $tn $tbl $expr $results] + } +} do_execsql_test 4.1.0 { - CREATE VIRTUAL TABLE t4 USING fts5(x, y); + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); } @@ -185,7 +195,7 @@ do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { xpxsscplax - } -do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { +do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} { p {1 1} c {2 2} x { @@ -203,8 +213,8 @@ do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { } do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } -do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } -do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } +do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { x { @@ -220,7 +230,7 @@ do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} { } do_execsql_test 4.2.0 { - CREATE VIRTUAL TABLE t5 USING fts5(content); + CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%); INSERT INTO t5 VALUES('a a a a a'); INSERT INTO t5 VALUES('a b a b a'); INSERT INTO t5 VALUES('c b c b c'); @@ -233,7 +243,7 @@ do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; @@ -250,7 +260,7 @@ if 0 { do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } @@ -260,10 +270,10 @@ do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } -do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } +do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_execsql_test 4.5.0 { - CREATE VIRTUAL TABLE t6 USING fts5(a, b, c); + CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%); INSERT INTO t6 VALUES('a', 'b', 'c'); } do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } @@ -274,7 +284,7 @@ do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } # use the full-text index (i.e. lookup by rowid or full-table scan). # do_execsql_test 7.1 { - CREATE VIRTUAL TABLE t10 USING fts5(content); + CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%); INSERT INTO t10 VALUES('first record'); INSERT INTO t10 VALUES('second record'); } @@ -299,7 +309,7 @@ do_execsql_test 7.4 { # SELECT sum(length(content)) < count(*) FROM fts4table; # do_execsql_test 8.1 { - CREATE VIRTUAL TABLE t11 USING fts5(content); + CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%); INSERT INTO t11(t11, rank) VALUES('pgsz', 32); INSERT INTO t11 VALUES('quitealongstringoftext'); INSERT INTO t11 VALUES('anotherquitealongstringoftext'); @@ -318,22 +328,24 @@ do_execsql_test 8.3 { #------------------------------------------------------------------------- -do_execsql_test 9.1 { - CREATE VIRTUAL TABLE t12 USING fts5(content); - INSERT INTO t12 VALUES('a b c d'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} {{0 1 1 0 1 1 1 1 1}} -do_execsql_test 9.2 { - INSERT INTO t12 VALUES('a d c d'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} { - {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} -} -do_execsql_test 9.3 { - INSERT INTO t12 VALUES('a d d a'); - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; -} { - {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} +if {[detail_is_full]} { + do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%); + INSERT INTO t12 VALUES('a b c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } {{0 1 1 0 1 1 1 1 1}} + do_execsql_test 9.2 { + INSERT INTO t12 VALUES('a d c d'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} + } + do_execsql_test 9.3 { + INSERT INTO t12 VALUES('a d d a'); + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; + } { + {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} + } } #--------------------------------------------------------------------------- @@ -341,7 +353,7 @@ do_execsql_test 9.3 { # do_execsql_test 10.1 { DROP TABLE t10; - CREATE VIRTUAL TABLE t10 USING fts5(idx, value); + CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%); INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); SELECT t10.rowid, t10.* FROM t10 @@ -358,7 +370,7 @@ do_execsql_test 10.1 { reset_db sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { - CREATE VIRTUAL TABLE tt USING fts5(x, y); + CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 @@ -410,6 +422,8 @@ foreach {tn expr res} { } } { + + if {[string match *:* $expr] && [detail_is_none]} continue do_execsql_test 11.1.$tn.1 { SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr } $res @@ -443,7 +457,7 @@ db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } - execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)" } {} do_execsql_test 12.1 { @@ -451,5 +465,7 @@ do_execsql_test 12.1 { SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] +} ;# foreach_detail_mode + finish_test diff --git a/manifest b/manifest index ff5d797e2d..25592e2931 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\smore\sfts5\stests\sto\srun\sin\sdetail=none\sand\sdetail=column\smodes\sas\swell\sas\sthe\sdefault\sdetail=full. -D 2016-01-04T16:19:51.421 +C Fix\san\sOOM\shandling\sproblem\sin\sfts5\sdetail=none\sand\sdetail=col\smodes.\sAlso\sa\sbug\sin\sthe\sxInst()\sAPI\swhen\sused\swith\sdetail=col\sand\scolumn\sfiltering\sexpressions.\sUpdate\sthe\smatchinfo()\stest\sfunction\sso\sthat\s'b'\sis\sfast\swith\sdetail=col\stables. +D 2016-01-04T19:12:00.230 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,17 +97,17 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 -F ext/fts5/fts5Int.h cff7dd3131ba0db6f699df97237bf993f22c1a1f +F ext/fts5/fts5Int.h 10608c346cccf7dd1da4d6b46f7921949072ed60 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 898e0bb53280b2c4c2702a2dc08dec731c7b32fe +F ext/fts5/fts5_expr.c 3fe372518c0191230971321048e10930952dad5b F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c -F ext/fts5/fts5_main.c f33439bde2e1023888e9b7f16e463e42fe4c00c5 +F ext/fts5/fts5_main.c 15c8b702e28d032224a6fc2b6a9c03ba2e4deeb1 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb -F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 +F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 @@ -117,7 +117,7 @@ F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824 F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b -F ext/fts5/test/fts5ac.test 9a3c5eb052a5e8c871d56d3d0f144e5fdcf075c2 +F ext/fts5/test/fts5ac.test 30707d2972e5f485f7bdef019f3041782edabfb3 F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c @@ -151,10 +151,11 @@ F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603 F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b +F ext/fts5/test/fts5fault8.test aeb4717b7b293678bc4d2f3c0159206a525375d9 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 -F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 +F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c @@ -1407,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6322a1d984e7946735bace8a069ef24b31754b3b -R 32e8694f89e6ec744ed1fb193a76a532 +P 3fcf3b1e24483b2cd1c1710f053ed8403e09106d +R 133d47de041747f0d7e918f07a474874 U dan -Z 780f62d5b61ae786fb811bd6c42602f7 +Z 4f94b859477e06c41fefb0b71cc6e55e diff --git a/manifest.uuid b/manifest.uuid index 3f2a04cfe4..1f4ad7217a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3fcf3b1e24483b2cd1c1710f053ed8403e09106d \ No newline at end of file +ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9 \ No newline at end of file From 10e1535dee1085a602c09e7847532e61057c7e4d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Jan 2016 21:04:35 +0000 Subject: [PATCH 13/44] Rationalize and extend some fts5 test code. Fix bugs revealed by the same. FossilOrigin-Name: fd217fdb396cf22278a496de0e9dec7902ba8b1f --- ext/fts5/fts5Int.h | 3 +- ext/fts5/fts5_expr.c | 34 +++- ext/fts5/fts5_main.c | 5 +- ext/fts5/test/fts5_common.tcl | 113 +++++++++++++ ext/fts5/test/fts5ac.test | 291 +++++++++++++--------------------- ext/fts5/test/fts5auto.test | 41 +---- ext/fts5/test/fts5simple.test | 21 +++ manifest | 24 +-- manifest.uuid | 2 +- 9 files changed, 299 insertions(+), 235 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index b853bb3a43..1a3d57ca26 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -643,11 +643,12 @@ int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; -Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*); +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); +void sqlite3Fts5ExprClearEof(Fts5Expr*); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ae00a12c82..9988c557b4 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1218,6 +1218,9 @@ static int fts5ExprNodeNextMatch( } pNode->bEof = p1->bEof; pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); + } break; } } @@ -2260,9 +2263,10 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ struct Fts5PoslistPopulator { Fts5PoslistWriter writer; int bOk; /* True if ok to populate */ + int bMiss; }; -Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ Fts5PoslistPopulator *pRet; pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); if( pRet ){ @@ -2270,8 +2274,15 @@ Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); for(i=0; inPhrase; i++){ Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; assert( pExpr->apExprPhrase[i]->nTerm==1 ); - pBuf->n = 0; + if( bLive && + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) + ){ + pRet[i].bMiss = 1; + }else{ + pBuf->n = 0; + } } } return pRet; @@ -2341,8 +2352,11 @@ int sqlite3Fts5ExprPopulatePoslists( sCtx.iOff = (((i64)iCol) << 32) - 1; for(i=0; inPhrase; i++){ - Fts5Colset *pColset = pExpr->apExprPhrase[i]->pNode->pNear->pColset; - if( pColset && 0==fts5ExprColsetTest(pColset, iCol) ){ + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; + Fts5Colset *pColset = pNode->pNear->pColset; + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + || aPopulator[i].bMiss + ){ aPopulator[i].bOk = 0; }else{ aPopulator[i].bOk = 1; @@ -2412,10 +2426,22 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ } } } + void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } +static void fts5ExprClearEof(Fts5ExprNode *pNode){ + int i; + for(i=0; inChild; i++){ + fts5ExprClearEof(pNode->apChild[i]); + } + pNode->bEof = 0; +} +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ + fts5ExprClearEof(pExpr->pRoot); +} + /* ** This function is only called for detail=columns tables. */ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 99c9eb05db..7f6f6dacef 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1179,6 +1179,7 @@ static int fts5FilterMethod( pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); + sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; @@ -1688,13 +1689,14 @@ static int fts5CsrPoslist( ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; + int bLive = (pCsr->pSorter==0); if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; - aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; for(i=0; inCol && rc==SQLITE_OK; i++){ int n; const char *z; @@ -1706,6 +1708,7 @@ static int fts5CsrPoslist( } } sqlite3_free(aPopulator); + if( pCsr->pSorter ){ sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 3bcd33dcfc..aadf148716 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -390,5 +390,118 @@ proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } +#------------------------------------------------------------------------- +# Convert a poslist of the type returned by fts5_test_poslist() to a +# collist as returned by fts5_test_collist(). +# +proc fts5_poslist2collist {poslist} { + set res [list] + foreach h $poslist { + regexp {(.*)\.[1234567890]+} $h -> cand + lappend res $cand + } + set res [lsort -command fts5_collist_elem_compare -unique $res] + return $res +} + +# Comparison function used by fts5_poslist2collist to sort collist entries. +# +proc fts5_collist_elem_compare {a b} { + foreach {a1 a2} [split $a .] {} + foreach {b1 b2} [split $b .] {} + + if {$a1==$b1} { return [expr $a2 - $b2] } + return [expr $a1 - $b1] +} +#-------------------------------------------------------------------------- +# Construct and return a tcl list equivalent to that returned by the SQL +# query executed against database handle [db]: +# +# SELECT +# rowid, +# fts5_test_poslist($tbl), +# fts5_test_collist($tbl) +# FROM $tbl('$expr') +# ORDER BY rowid $order; +# +proc fts5_query_data {expr tbl {order ASC}} { + + # Figure out the set of columns in the FTS5 table. This routine does + # not handle tables with UNINDEXED columns, but if it did, it would + # have to be here. + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, 'nearset $cols -pc ::pc' [set cols] ) + }]] + + set res [list] + db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { + set cols [list] + foreach col $lCols { lappend cols $x($col) } + + set ::pc 0 + set rowdata [eval $tclexpr] + if {$rowdata != ""} { + lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata] + } + } + + set res +} + +#------------------------------------------------------------------------- +# Similar to [fts5_query_data], but omit the collist field. +# +proc fts5_poslist_data {expr tbl {order ASC}} { + set res [list] + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order] { + lappend res $rowid $poslist + } + set res +} + +#------------------------------------------------------------------------- +# +proc nearset_rf {aCol args} { + set idx [lsearch -exact $args --] + if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { + set ::expr_not_ok 1 + } + list +} + +proc nearset_rc {aCol args} { + nearset_rf $aCol {*}$args + if {[lsearch $args -col]>=0} { + set ::expr_not_ok 1 + } + list +} + +proc fts5_expr_ok {expr tbl} { + + if {![detail_is_full]} { + set nearset "nearset_rc" + if {[detail_is_col]} { set nearset "nearset_rf" } + + set ::expr_not_ok 0 + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + + set cols "" + foreach e $lCols { append cols ", '$e'" } + set ::pc 0 + set tclexpr [db one [subst -novar { + SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] ) + }]] + eval $tclexpr + if {$::expr_not_ok} { return 0 } + } + + return 1 +} + diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 692929167b..f53f5a4622 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -211,6 +211,7 @@ foreach {tn2 sql} { 1 {} 2 {BEGIN} } { +if {[detail_is_full]} continue reset_db fts5_aux_test_functions db @@ -228,191 +229,114 @@ foreach {tn2 sql} { execsql { INSERT INTO xx(xx) VALUES('integrity-check') } } {} - #------------------------------------------------------------------------- - # The following work with all detail= modes. - # - foreach {tn expr} { - 1 "a AND b" - 2 "a OR b" - 3 "o" - 4 "b q" - 5 "e a e" - 6 "m d g q q b k b w f q q p p" - 7 "l o o l v v k" - 8 "a" - 9 "b" - 10 "c" - 11 "no" - 12 "L O O L V V K" - 13 "a AND b AND c" - } { - set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.1.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.1.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - - if {[detail_is_none]} continue - - #------------------------------------------------------------------------- - # Queries on a specific column. - # - foreach {tn expr} { - 1.1 "x:a" - 1.2 "y:a" - 1.3 "x:b" - 1.4 "y:b" - 2.1 "{x}:a" - 2.2 "{y}:a" - 2.3 "{x}:b" - 2.4 "{y}:b" - - 3.1 "{x y}:a" - 3.2 "{y x}:a" - 3.3 "{x x}:b" - 3.4 "{y y}:b" - - 4.1 {{"x" "y"}:a} - 4.2 {{"y" x}:a} - 4.3 {{x "x"}:b} - 4.4 {{"y" y}:b} - } { - set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - - if {[detail_is_col]} continue - - #------------------------------------------------------------------------- - # Test phrase queries. - # - foreach {tn expr} { - 1 "b + q" - 2 "e + a + e" - 3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" - 4 "l + o + o + l + v + v + k" - 5 "L + O + O + L + V + V + K" - } { - set res [poslist_data 1 $expr] - - do_execsql_test 1.$tn2.2.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - - #------------------------------------------------------------------------- - # Test some AND and OR queries. - # - foreach {tn expr} { - 1.1 "a+b AND c" - 1.2 "d+c AND u" - 1.3 "d+c AND u+d" - - 2.1 "a+b OR c" - 2.2 "d+c OR u" - 2.3 "d+c OR u+d" - } { - set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - - #------------------------------------------------------------------------- - # Some NEAR queries. - # - foreach {tn expr} { - 1 "NEAR(a b)" - 2 "NEAR(r c)" - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } - } { - set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.5.1.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res - - set res [collist_data $expr] - do_execsql_test 1.$tn2.5.1.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr - } $res - } - - do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0} - do_test $tn2.4.2 { nearset {{a b c}} -- c } {0.0.2} - - foreach {tn expr tclexpr} { - 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} - } { - do_execsql_test 1.$tn2.6.$tn { - SELECT fts5_expr_tcl($expr, 'N $x') - } [list $tclexpr] - } #------------------------------------------------------------------------- # - do_execsql_test 1.$tn2.7.integrity { + do_execsql_test 1.$tn2.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } - #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} - foreach {bAsc sql} { - 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} - 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} + + #------------------------------------------------------------------------- + # + foreach {tn expr} { + 1.2 "a OR b" + 1.1 "a AND b" + 1.3 "o" + 1.4 "b q" + 1.5 "e a e" + 1.6 "m d g q q b k b w f q q p p" + 1.7 "l o o l v v k" + 1.8 "a" + 1.9 "b" + 1.10 "c" + 1.11 "no" + 1.12 "L O O L V V K" + 1.13 "a AND b AND c" + 1.14 "x:a" + + 2.1 "x:a" + 2.2 "y:a" + 2.3 "x:b" + 2.4 "y:b" + + 3.1 "{x}:a" + 3.2 "{y}:a" + 3.3 "{x}:b" + 3.4 "{y}:b" + + 4.1 "{x y}:a" + 4.2 "{y x}:a" + 4.3 "{x x}:b" + 4.4 "{y y}:b" + + 5.1 {{"x" "y"}:a} + 5.2 {{"y" x}:a} + 5.3 {{x "x"}:b} + 5.4 {{"y" y}:b} + + 6.1 "b + q" + 6.2 "e + a + e" + 6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" + 6.4 "l + o + o + l + v + v + k" + 6.5 "L + O + O + L + V + V + K" + + 7.1 "a+b AND c" + 7.2 "d+c AND u" + 7.3 "d+c AND u+d" + 7.4 "a+b OR c" + 7.5 "d+c OR u" + 7.6 "d+c OR u+d" + + 8.1 "NEAR(a b)" + 8.2 "NEAR(r c)" + 8.2 { NEAR(r c, 5) } + 8.3 { NEAR(r c, 3) } + 8.4 { NEAR(r c, 2) } + 8.5 { NEAR(r c, 0) } + 8.6 { NEAR(a b c) } + 8.7 { NEAR(a b c, 8) } + 8.8 { x : NEAR(r c) } + 8.9 { y : NEAR(r c) } + + 9.1 { NEAR(r c) } + 9.2 { NEAR(r c, 5) } + 9.3 { NEAR(r c, 3) } + 9.4 { NEAR(r c, 2) } + 9.5 { NEAR(r c, 0) } + 9.6 { NEAR(a b c) } + 9.7 { NEAR(a b c, 8) } + 9.8 { x : NEAR(r c) } + 9.9 { y : NEAR(r c) } + 9.10 { x : "r c" } + 9.11 { y : "r c" } + 9.12 { a AND b } + 9.13 { a AND b AND c } + 9.14a { a } + 9.14b { a OR b } + 9.15 { a OR b AND c } + 9.16 { c AND b OR a } + 9.17 { c AND (b OR a) } + 9.18 { c NOT (b OR a) } + 9.19 { (c NOT b) OR (a AND d) } } { - foreach {tn expr} { - 0.1 x - 1 { NEAR(r c) } - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } - 10 { x : "r c" } - 11 { y : "r c" } - 12 { a AND b } - 13 { a AND b AND c } - 14a { a } - 14b { a OR b } - 15 { a OR b AND c } - 16 { c AND b OR a } - 17 { c AND (b OR a) } - 18 { c NOT (b OR a) } - 19 { c NOT b OR a AND d } - } { - set res [poslist_data 0 $expr $bAsc] - do_execsql_test 1.$tn2.7.$bAsc.$tn.[llength $res] $sql $res + + if {[fts5_expr_ok $expr xx]==0} { + do_test 1.$tn2.$tn.OMITTED { list } [list] + continue } + + set res [fts5_query_data $expr xx] + do_execsql_test 1.$tn2.$tn.[llength $res].asc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr + } $res + + + set res [fts5_query_data $expr xx DESC] + do_execsql_test 1.$tn2.$tn.[llength $res].desc { + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) + FROM xx WHERE xx match $expr ORDER BY 1 DESC + } $res } } @@ -422,7 +346,16 @@ do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} -finish_test +do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0} +do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2} + +foreach {tn expr tclexpr} { + 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} +} { + do_execsql_test 2.3.$tn { + SELECT fts5_expr_tcl($expr, 'N $x') + } [list $tclexpr] +} finish_test diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test index 6efaa9cb7c..218b3f4862 100644 --- a/ext/fts5/test/fts5auto.test +++ b/ext/fts5/test/fts5auto.test @@ -22,7 +22,6 @@ ifcapable !fts5 { return } - set data { -4026076 {n x w k b p x b n t t d s} {f j j s p j o} @@ -232,37 +231,9 @@ do_execsql_test 1.0 { fts5_aux_test_functions db -proc matchdata {expr tbl collist {order ASC}} { - - set cols "" - foreach e $collist { - append cols ", '$e'" - } - - set tclexpr [db one [subst -novar { - SELECT fts5_expr_tcl( - $expr, 'nearset $cols -pc ::pc' [set cols] - ) - }]] - set res [list] - - db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { - set cols [list] - foreach col $x(*) { - if {$col != "rowid"} { lappend cols $x($col) } - } - # set cols [list $a $b $c $d $e $f] - set ::pc 0 - set rowdata [eval $tclexpr] - if {$rowdata != ""} { lappend res $x(rowid) $rowdata } - } - - set res -} - -proc do_auto_test {tn tbl cols expr} { +proc do_auto_test {tn tbl expr} { foreach order {asc desc} { - set res [matchdata $expr $tbl $cols $order] + set res [fts5_poslist_data $expr $tbl $order] set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]" set ::autotest_expr $expr @@ -271,8 +242,6 @@ proc do_auto_test {tn tbl cols expr} { WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order] }] $res } - - } #------------------------------------------------------------------------- @@ -332,7 +301,7 @@ for {set fold 0} {$fold < 3} {incr fold} { C.1 { a OR (b AND "b c") } C.2 { a OR (b AND "z c") } } { - do_auto_test 3.$fold.$tn tt {a b c d e f} $expr + do_auto_test 3.$fold.$tn tt $expr } } @@ -366,10 +335,8 @@ foreach {tn expr} { 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} - - } { - do_auto_test 4.$tn yy {c1 c2 c3} $expr + do_auto_test 4.$tn yy $expr } diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 870a912bb1..e29c13117c 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -19,6 +19,8 @@ ifcapable !fts5 { return } + if 1 { + #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" @@ -361,5 +363,24 @@ do_execsql_test 15.1 { INSERT INTO x2(x2) VALUES('integrity-check'); } +} + +#------------------------------------------------------------------------- +foreach_detail_mode $testprefix { + reset_db + fts5_aux_test_functions db + do_execsql_test 16.0 { + CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); + INSERT INTO x3 VALUES('a b c d e f'); + } + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); + } {2.0.2} + + do_execsql_test 16.1 { + SELECT fts5_test_poslist(x3) FROM x3('a OR c'); + } {{0.0.0 1.0.2}} +} + finish_test diff --git a/manifest b/manifest index 25592e2931..0ccb6b20cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sOOM\shandling\sproblem\sin\sfts5\sdetail=none\sand\sdetail=col\smodes.\sAlso\sa\sbug\sin\sthe\sxInst()\sAPI\swhen\sused\swith\sdetail=col\sand\scolumn\sfiltering\sexpressions.\sUpdate\sthe\smatchinfo()\stest\sfunction\sso\sthat\s'b'\sis\sfast\swith\sdetail=col\stables. -D 2016-01-04T19:12:00.230 +C Rationalize\sand\sextend\ssome\sfts5\stest\scode.\sFix\sbugs\srevealed\sby\sthe\ssame. +D 2016-01-05T21:04:35.294 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,14 +97,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 -F ext/fts5/fts5Int.h 10608c346cccf7dd1da4d6b46f7921949072ed60 +F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 3fe372518c0191230971321048e10930952dad5b +F ext/fts5/fts5_expr.c 1aa0f5d597a22789172efe138659271fccb3cf19 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c -F ext/fts5/fts5_main.c 15c8b702e28d032224a6fc2b6a9c03ba2e4deeb1 +F ext/fts5/fts5_main.c cd5f6f0bf868a19e8f8ef01c7b4b86dd152f0bc7 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 @@ -114,10 +114,10 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824 +F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b -F ext/fts5/test/fts5ac.test 30707d2972e5f485f7bdef019f3041782edabfb3 +F ext/fts5/test/fts5ac.test 63f669ad7381b0c29556da6de41eefdbb007e810 F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c @@ -128,7 +128,7 @@ F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9 F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 -F ext/fts5/test/fts5auto.test 2a6241673657b340427f521528f7809ddaa02a9e +F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb @@ -170,7 +170,7 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 -F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 +F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test d453e3ca100e08078a46b12547417891b326240a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 @@ -1408,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3fcf3b1e24483b2cd1c1710f053ed8403e09106d -R 133d47de041747f0d7e918f07a474874 +P ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9 +R 6eca7a3ddda49a17d0ee2479d97ee402 U dan -Z 4f94b859477e06c41fefb0b71cc6e55e +Z 551e607b90efc1e18e4dc205bbcdae9f diff --git a/manifest.uuid b/manifest.uuid index 1f4ad7217a..49ed937a10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9 \ No newline at end of file +fd217fdb396cf22278a496de0e9dec7902ba8b1f \ No newline at end of file From f062d676ef28e51ec9567cf3a4fd896005e7667b Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jan 2016 10:17:26 +0000 Subject: [PATCH 14/44] Fix a problem with the xPhraseFirstColumn() API in detail=col mode. Remove some unused test code from fts5ac.test. FossilOrigin-Name: a4e55f73b3c81e28b917395089c0201f9a2c9178 --- ext/fts5/fts5_expr.c | 5 ++- ext/fts5/test/fts5ac.test | 81 --------------------------------------- manifest | 14 +++---- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 90 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9988c557b4..c0dd0ec066 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2455,7 +2455,10 @@ int sqlite3Fts5ExprPhraseCollist( Fts5ExprNode *pNode = pPhrase->pNode; assert( iPhrase>=0 && iPhrasenPhrase ); - if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid + && pPhrase->poslist.n>0 + ){ sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); }else{ *ppCollist = 0; diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index f53f5a4622..00b1328867 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -127,91 +127,10 @@ set data { 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} } -# Argument $expr is an FTS5 match expression designed to be executed against -# an FTS5 table with the following schema: -# -# CREATE VIRTUAL TABLE xy USING fts5(x, y); -# -# Assuming the table contains the same records as stored in the global -# $::data array (see above), this function returns a list containing two -# elements for each matching row in the dataset. The first element of each -# pair is the rowid. The second is a list of phrase matches, where each -# phrase match is of the form: -# -# .. -# -# The list of phrase matches is in the same format as that returned by the -# fts5_test_poslist() auxiliary scalara function. -# -proc poslist_data {bPos expr {bAsc 1}} { - - set tclexpr [db one { - SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y') - }] - set res [list] - - #puts $tclexpr - foreach {id x y} $::data { - set cols [list $x $y] - set ::pc 0 - #set hits [lsort -command instcompare [eval $tclexpr]] - set hits [eval $tclexpr] - if {[llength $hits]>0} { - if {$bPos} { - lappend res [list $id $hits] - } else { - lappend res $id - } - } - } - - if {$bAsc} { - set res [lsort -integer -increasing -index 0 $res] - } else { - set res [lsort -integer -decreasing -index 0 $res] - } - - set res [concat {*}$res] - return $res -} - -proc collist_elem_compare {a b} { - foreach {a1 a2} [split $a .] {} - foreach {b1 b2} [split $b .] {} - - if {$a1==$b1} { - return [expr $a2 - $b2] - } - return [expr $a1 - $b1] -} - -proc poslist2collist {poslist} { - set res [list] - foreach h $poslist { - regexp {(.*)\.[1234567890]+} $h -> cand - lappend res $cand - } - set res [lsort -command collist_elem_compare -unique $res] - return $res -} - -proc collist_data {expr} { - set res [list] - foreach {rowid poslist} [poslist_data 1 $expr] { - lappend res $rowid [poslist2collist $poslist] - } - set res -} - -# -# End of test code -#------------------------------------------------------------------------- - foreach {tn2 sql} { 1 {} 2 {BEGIN} } { -if {[detail_is_full]} continue reset_db fts5_aux_test_functions db diff --git a/manifest b/manifest index 0ccb6b20cc..e95f7f880f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rationalize\sand\sextend\ssome\sfts5\stest\scode.\sFix\sbugs\srevealed\sby\sthe\ssame. -D 2016-01-05T21:04:35.294 +C Fix\sa\sproblem\swith\sthe\sxPhraseFirstColumn()\sAPI\sin\sdetail=col\smode.\sRemove\ssome\sunused\stest\scode\sfrom\sfts5ac.test. +D 2016-01-06T10:17:26.540 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -101,7 +101,7 @@ F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 1aa0f5d597a22789172efe138659271fccb3cf19 +F ext/fts5/fts5_expr.c 5d5dd3044c1a39ace6ddd1034961fd5278ffce2d F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c F ext/fts5/fts5_main.c cd5f6f0bf868a19e8f8ef01c7b4b86dd152f0bc7 @@ -117,7 +117,7 @@ F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b -F ext/fts5/test/fts5ac.test 63f669ad7381b0c29556da6de41eefdbb007e810 +F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c @@ -1408,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9 -R 6eca7a3ddda49a17d0ee2479d97ee402 +P fd217fdb396cf22278a496de0e9dec7902ba8b1f +R fb939ae0b5d34f9dd8275713b2d3d9d0 U dan -Z 551e607b90efc1e18e4dc205bbcdae9f +Z ffed0e3dee313fbcc0c638ddfdf4d418 diff --git a/manifest.uuid b/manifest.uuid index 49ed937a10..57c2bbbbf5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd217fdb396cf22278a496de0e9dec7902ba8b1f \ No newline at end of file +a4e55f73b3c81e28b917395089c0201f9a2c9178 \ No newline at end of file From 4364e5c9f760cc6335cc49de9257e1de8c7e1424 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jan 2016 18:53:57 +0000 Subject: [PATCH 15/44] Fix an fts5 problem involving detail=none, "ORDER BY rowid DESC" and deleted items. Also add tests to verify that the documented operator precedences are correct. FossilOrigin-Name: 8d05cfd48db1462dfd3efd502d53bd33214403fb --- ext/fts5/fts5_index.c | 8 ++++---- ext/fts5/test/fts5dlidx.test | 25 +++++++++++++----------- ext/fts5/test/fts5eb.test | 9 +++++++++ ext/fts5/test/fts5simple2.test | 35 ++++++++++++++++++++++++++++++++-- manifest | 18 ++++++++--------- manifest.uuid | 2 +- 6 files changed, 70 insertions(+), 27 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 49d8638f33..ef673085c6 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1764,14 +1764,14 @@ static void fts5SegIterNext( i64 iDelta; pIter->iRowidOffset--; - pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ - iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); - iOff += nPos; + iOff += pIter->nPos; } fts5GetVarint(&a[iOff], (u64*)&iDelta); pIter->iRowid -= iDelta; - fts5SegIterLoadNPos(p, pIter); }else{ fts5SegIterReverseNewPage(p, pIter); } diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 5a03c6989b..232b5021f1 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -26,7 +26,7 @@ if { $tcl_platform(wordSize)<8 } { return } -if 1 { +foreach_detail_mode $testprefix { proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] @@ -34,7 +34,7 @@ proc do_fb_test {tn sql res} { uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] } -# This test populates the FTS5 table containing $nEntry entries. Rows are +# This test populates the FTS5 table with $nEntry entries. Rows are # numbered from 0 to ($nEntry-1). The rowid for row $i is: # # ($iFirst + $i*$nStep) @@ -77,10 +77,12 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} { do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc - do_fb_test $tn.5.1 { - SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc - do_fb_test $tn.5.2 { - SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc + if {[detail_is_full]} { + do_fb_test $tn.5.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc + do_fb_test $tn.5.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc + } } @@ -90,7 +92,7 @@ foreach {tn pgsz} { } { do_execsql_test $tn.0 { DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); } @@ -107,7 +109,7 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { execsql { BEGIN; DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 64); INSERT INTO t1 VALUES('b a'); @@ -130,8 +132,6 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] -} - #-------------------------------------------------------------------- # reset_db @@ -158,7 +158,7 @@ proc rnddoc {} { db func rnddoc rnddoc do_execsql_test 3.1 { - CREATE VIRTUAL TABLE abc USING fts5(a); + CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%); INSERT INTO abc(abc, rank) VALUES('pgsz', 32); INSERT INTO abc VALUES ( rnddoc() ); @@ -192,6 +192,9 @@ foreach v $vocab { } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } +} ;# foreach_detail_mode + + finish_test diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 55c4b15cf3..8205396047 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -42,6 +42,15 @@ foreach {tn expr res} { 10 {abc + "" + def} {"abc" + "def"} 11 {abc "" def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} + + 13 {a AND b NOT c} {"a" AND ("b" NOT "c")} + 14 {a OR b NOT c} {"a" OR ("b" NOT "c")} + 15 {a NOT b AND c} {("a" NOT "b") AND "c"} + 16 {a NOT b OR c} {("a" NOT "b") OR "c"} + + 17 {a AND b OR c} {("a" AND "b") OR "c"} + 18 {a OR b AND c} {"a" OR ("b" AND "c")} + } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index bb76250e9f..c61970e154 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -254,8 +254,6 @@ foreach {rowid a} { } } -} - #------------------------------------------------------------------------- # reset_db @@ -269,5 +267,38 @@ do_execsql_test 14.1 { SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; } {0.0.1} +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(1, 'sqlite'); + INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); + COMMIT; +} {} + +do_test 15.1 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + +do_test 15.2 { + execsql { DELETE FROM t1 } +} {} + +do_execsql_test 15.3.1 { + SELECT rowid FROM t1('sqlite'); +} {} + +do_execsql_test 15.3.2 { + SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC; +} {} + +do_test 15.4 { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } +} {} + finish_test diff --git a/manifest b/manifest index e95f7f880f..39b91273fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\sxPhraseFirstColumn()\sAPI\sin\sdetail=col\smode.\sRemove\ssome\sunused\stest\scode\sfrom\sfts5ac.test. -D 2016-01-06T10:17:26.540 +C Fix\san\sfts5\sproblem\sinvolving\sdetail=none,\s"ORDER\sBY\srowid\sDESC"\sand\sdeleted\sitems.\sAlso\sadd\stests\sto\sverify\sthat\sthe\sdocumented\soperator\sprecedences\sare\scorrect. +D 2016-01-06T18:53:57.055 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 F ext/fts5/fts5_expr.c 5d5dd3044c1a39ace6ddd1034961fd5278ffce2d F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c +F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1 F ext/fts5/fts5_main.c cd5f6f0bf868a19e8f8ef01c7b4b86dd152f0bc7 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb @@ -140,10 +140,10 @@ F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d -F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d +F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 +F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 @@ -171,7 +171,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 -F ext/fts5/test/fts5simple2.test d453e3ca100e08078a46b12547417891b326240a +F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1408,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fd217fdb396cf22278a496de0e9dec7902ba8b1f -R fb939ae0b5d34f9dd8275713b2d3d9d0 +P a4e55f73b3c81e28b917395089c0201f9a2c9178 +R 0d0b45c96327e08ac49c0cd0bf01187b U dan -Z ffed0e3dee313fbcc0c638ddfdf4d418 +Z 1ffb33ee956b9f5de9322a26ad6982ed diff --git a/manifest.uuid b/manifest.uuid index 57c2bbbbf5..1b603ff0f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e55f73b3c81e28b917395089c0201f9a2c9178 \ No newline at end of file +8d05cfd48db1462dfd3efd502d53bd33214403fb \ No newline at end of file From 7c230dd99c1395982a9a4b75d6b4ef2eb75c3f77 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jan 2016 19:43:26 +0000 Subject: [PATCH 16/44] Fix a test script problem caused by a change in constraint handling within the core. FossilOrigin-Name: 625695b3d7d4eae7536b5f761b96cd7519ac6ada --- ext/fts5/test/fts5aa.test | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index e00bcfcdcc..fdcf08398d 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -424,11 +424,11 @@ do_execsql_test 16.1 { } proc funk {} { + db eval { UPDATE n1_config SET v=50 WHERE k='version' } set fd [db incrblob main n1_data block 10] fconfigure $fd -encoding binary -translation binary puts -nonewline $fd "\x44\x45" close $fd - db eval { UPDATE n1_config SET v=50 WHERE k='version' } } db func funk funk diff --git a/manifest b/manifest index b6ae4ac818..efcebbdb14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2016-01-06T19:35:52.285 +C Fix\sa\stest\sscript\sproblem\scaused\sby\sa\schange\sin\sconstraint\shandling\swithin\sthe\score. +D 2016-01-06T19:43:26.257 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -115,7 +115,7 @@ F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 -F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc +F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc @@ -1409,7 +1409,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8d05cfd48db1462dfd3efd502d53bd33214403fb 55a11fd627eaf046b3783622b2925399a50a65b4 -R cf1317de6526e81224b7c381ca0d373b +P e7dcd013516723941c2fb78ecdc96806d49d310b +R dce0c272fbae692a97296955b7b2c775 U dan -Z 2530c4785808f691eed775076fac89d0 +Z 318fda862dd5c3483c784e4e425f0506 diff --git a/manifest.uuid b/manifest.uuid index d32e744eb2..298843ae92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7dcd013516723941c2fb78ecdc96806d49d310b \ No newline at end of file +625695b3d7d4eae7536b5f761b96cd7519ac6ada \ No newline at end of file From 2598b833f06bbf6f8184ea74110ccd427fc8795c Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Jan 2016 20:07:41 +0000 Subject: [PATCH 17/44] Update the fts5vocab module to work with detail=col and detail=none tables. FossilOrigin-Name: eedd095dc1c81ce45df00093ba237dd7b3cdff3d --- ext/fts5/fts5_vocab.c | 86 +++++++++++++++++++++++---------- ext/fts5/test/fts5vocab.test | 92 ++++++++++++++++++++++++++---------- manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 134 insertions(+), 60 deletions(-) diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 860cfedb9b..2ed10b7cdb 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -379,7 +379,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ - if( pCsr->aCnt[pCsr->iCol] ) break; + if( pCsr->aDoc[pCsr->iCol] ) break; } } @@ -412,24 +412,52 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ - rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); - if( rc==SQLITE_OK ){ - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; - } - pCsr->aDoc[0]++; - }else{ - int iCol = -1; - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; - if( iCol!=ii ){ - pCsr->aDoc[ii]++; - iCol = ii; + switch( pCsr->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); + if( rc==SQLITE_OK ){ + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + pCsr->aDoc[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + pCsr->aDoc[ii]++; + iCol = ii; + } + } } } - } + break; + + case FTS5_DETAIL_COLUMNS: + if( pTab->eType==FTS5_VOCAB_ROW ){ + pCsr->aDoc[0]++; + }else{ + Fts5Buffer buf = {0, 0, 0}; + rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); + if( rc==SQLITE_OK ){ + while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPosaDoc[iPos]++; + } + } + sqlite3Fts5BufferFree(&buf); + } + break; + + default: + assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + break; + } + + if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } @@ -445,7 +473,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ - while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++; + while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; assert( pCsr->iColpConfig->nCol ); } return rc; @@ -525,30 +553,36 @@ static int fts5VocabColumnMethod( int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; if( iCol==0 ){ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); - } - else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){ + }else if( eType==FTS5_VOCAB_COL ){ assert( iCol==1 || iCol==2 || iCol==3 ); if( iCol==1 ){ - const char *z = pCsr->pConfig->azCol[pCsr->iCol]; - sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } }else if( iCol==2 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]); + iVal = pCsr->aDoc[pCsr->iCol]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]); + iVal = pCsr->aCnt[pCsr->iCol]; } }else{ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[0]); + iVal = pCsr->aDoc[0]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[0]); + iVal = pCsr->aCnt[0]; } } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; } diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index dc5099c6e3..1f5fa91663 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -21,9 +21,44 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + +if {[detail_is_none]} continue + +proc null_list_entries {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc null_insert {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc star_from_row {L} { + if {[detail_is_full]==0} { + set L [null_list_entries 2 3 $L] + } + return $L +} + +proc star_from_col {L} { + if {[detail_is_col]} { + set L [null_list_entries 3 4 $L] + } + if {[detail_is_none]} { + set L [null_list_entries 1 4 $L] + set L [null_list_entries 3 4 $L] + } + return $L +} do_execsql_test 1.1.1 { - CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); PRAGMA table_info = v1; } { @@ -52,32 +87,32 @@ do_execsql_test 1.3 { do_execsql_test 1.4.1 { SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] do_execsql_test 1.4.2 { SELECT * FROM v2; -} {x one 2 4 y one 1 1 z one 1 1} +} [star_from_col {x one 2 4 y one 1 1 z one 1 1}] do_execsql_test 1.5.1 { BEGIN; INSERT INTO t1 VALUES('a b c'); SELECT * FROM v1 WHERE term<'d'; -} {a 1 1 b 1 1 c 1 1} +} [star_from_row {a 1 1 b 1 1 c 1 1}] do_execsql_test 1.5.2 { SELECT * FROM v2 WHERE term<'d'; COMMIT; -} {a one 1 1 b one 1 1 c one 1 1} +} [star_from_col {a one 1 1 b one 1 1 c one 1 1}] do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE tt USING fts5(a, b); + CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); @@ -90,7 +125,12 @@ do_execsql_test 2.0 { INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); } -set res_col { +set res_row [star_from_row { + a 10 20 b 9 14 c 9 20 d 9 19 + e 8 13 f 10 20 g 7 14 x 1 1 + y 1 1 +}] +set res_col [star_from_col { a a 6 11 a b 7 9 b a 6 7 b b 7 7 c a 6 12 c b 5 8 @@ -99,12 +139,7 @@ set res_col { f a 9 10 f b 7 10 g a 5 7 g b 5 7 x a 1 1 y b 1 1 -} -set res_row { - a 10 20 b 9 14 c 9 20 d 9 19 - e 8 13 f 10 20 g 7 14 x 1 1 - y 1 1 -} +}] foreach {tn tbl resname} { 1 "fts5vocab(tt, 'col')" res_col @@ -153,9 +188,9 @@ reset_db forcedelete test.db2 do_execsql_test 5.0 { ATTACH 'test.db2' AS aux; - CREATE VIRTUAL TABLE t1 USING fts5(x); - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); - CREATE VIRTUAL TABLE aux.t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO main.t1 VALUES('a b c'); INSERT INTO main.t1 VALUES('d e f'); @@ -178,18 +213,18 @@ do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); } -do_execsql_test 5.2 { SELECT * FROM vm } { +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 -} -do_execsql_test 5.3 { SELECT * FROM vt1 } { +}] +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.4 { SELECT * FROM vt2 } { +}] +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.5 { SELECT * FROM va } { +}] +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 -} +}] #------------------------------------------------------------------------- # @@ -218,7 +253,7 @@ do_catchsql_test 6.2 { # constraints in the WHERE clause). # do_execsql_test 7.0 { - CREATE VIRTUAL TABLE tx USING fts5(one, two); + CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); @@ -276,6 +311,9 @@ foreach {term} { if {[lindex $r2 2]==0} {set r2 [list]} set resc [concat $r1 $r2] + + set resc [star_from_col $resc] + set resr [star_from_row $resr] do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } @@ -345,5 +383,7 @@ do_execsql_test 7.3.2 { WHERE txc.term = txc_c.term AND txc.col=txc_c.col; } {57} +} + finish_test diff --git a/manifest b/manifest index efcebbdb14..d860431936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\scaused\sby\sa\schange\sin\sconstraint\shandling\swithin\sthe\score. -D 2016-01-06T19:43:26.257 +C Update\sthe\sfts5vocab\smodule\sto\swork\swith\sdetail=col\sand\sdetail=none\stables. +D 2016-01-07T20:07:41.638 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -111,7 +111,7 @@ F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 +F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 @@ -179,7 +179,7 @@ F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0 +F ext/fts5/test/fts5vocab.test f4ff3ba4036a83678f2a8ba8b89eb303bebc122e F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -1409,7 +1409,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e7dcd013516723941c2fb78ecdc96806d49d310b -R dce0c272fbae692a97296955b7b2c775 +P 625695b3d7d4eae7536b5f761b96cd7519ac6ada +R 276342437a8f737c3f32b7f1a4e61d27 U dan -Z 318fda862dd5c3483c784e4e425f0506 +Z 8c87f6adf59add172dd12da497a3ce1f diff --git a/manifest.uuid b/manifest.uuid index 298843ae92..46d1607a71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -625695b3d7d4eae7536b5f761b96cd7519ac6ada \ No newline at end of file +eedd095dc1c81ce45df00093ba237dd7b3cdff3d \ No newline at end of file From 21d61853e741bae5cc2d89c1ead99431b9e6454e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jan 2016 02:27:01 +0000 Subject: [PATCH 18/44] Add the experimental SQLITE_FCNTL_JOURNAL_POINTER file control for obtaining the sqlite3_file pointer associated with a rollback journal or WAL file. FossilOrigin-Name: a02ace9a746ff02fc1a5969bf13939b568624f93 --- manifest | 25 ++++++++++++++----------- manifest.uuid | 2 +- src/main.c | 3 +++ src/pager.c | 8 ++++++++ src/pager.h | 1 + src/sqlite.h.in | 1 + src/wal.c | 6 ++++++ src/wal.h | 4 ++++ 8 files changed, 38 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index fbb4b40d59..183b7bbcf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sperformance\stracking\sfrom\sthe\smemsys5\smemory\sallocator\sif\sneither\nSQLITE_DEBUG\snor\sSQLITE_TEST\sare\sdefined. -D 2016-01-07T21:12:40.900 +C Add\sthe\sexperimental\sSQLITE_FCNTL_JOURNAL_POINTER\sfile\scontrol\sfor\sobtaining\nthe\ssqlite3_file\spointer\sassociated\swith\sa\srollback\sjournal\sor\sWAL\sfile. +D 2016-01-08T02:27:01.544 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -298,7 +298,7 @@ F src/insert.c 6b45cc86967da11aa024d034745f93f66d53f650 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b -F src/main.c b9641a4f650230bfc5dbb0bebf7b33ff4d27586a +F src/main.c 6d589f27980c4aec014ae9a081d611e7e5ebc115 F src/malloc.c 8f787669e79de26efc42272b5797bc00fff527c6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -320,8 +320,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 0ca6d8710366fbb01a275160f018334cd347cbda F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 58d2593612acb6b542de6715b4af397ea1fa0a35 -F src/pager.h bf25005b4656cd805af43487c3139fca9678d0cc +F src/pager.c 5dda328a9a5e9405b6971d62d0d5e3ea85c6195a +F src/pager.h 1c2a49143dfba9e69cc8159ef019f472ed8d260b F src/parse.y caad1e98edeca6960493d0c60d31b76820dd7776 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 F src/pcache.h 1ff11adce609ba7de139b6abfabaf9a2bac947b5 @@ -335,7 +335,7 @@ F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f8fded11fc443a9f5a73cc5db069d06b34460e2f F src/shell.c ed71dc7679e6f087a3f1ea3f9dae4b0fae7209c3 -F src/sqlite.h.in 7d87d71b9a4689c51fa092f48f16590ff71558e3 +F src/sqlite.h.in f18858fc9440fd3c399b733057d12c53a2360d11 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d F src/sqliteInt.h 987b46b1d8089155caecbdb14b78850ae1e198ff @@ -409,8 +409,8 @@ F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 974928c988681c5157202c79dd9f26afaa7b5086 -F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b +F src/wal.c 407f6f3929aeeb360f53a9279342940fe79823e1 +F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c c6d3d2f6af57d574a7365ee2b225a5024f2a6bec F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a @@ -1406,7 +1406,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2d96aeba2460779a0a20356739a0ba49144c8a85 -R 0f87ccc97eed432e32f05b109bfdb1a7 +P af5c7714e993f060841f1e893f754ddf3870e6d0 +R 1ed70ce93210d32572be9b1ea5c549df +T *branch * file-control-journal +T *sym-file-control-journal * +T -sym-trunk * U drh -Z 9752f7a1ad4b146c9e3d9fcb5efbd790 +Z 34b65576034eb21a6f6ec3e5456f3b6a diff --git a/manifest.uuid b/manifest.uuid index 9cc6b8eedd..8d6b249f6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af5c7714e993f060841f1e893f754ddf3870e6d0 \ No newline at end of file +a02ace9a746ff02fc1a5969bf13939b568624f93 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 8b75e71500..2998f54c77 100644 --- a/src/main.c +++ b/src/main.c @@ -3424,6 +3424,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ }else if( op==SQLITE_FCNTL_VFS_POINTER ){ *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); }else{ diff --git a/src/pager.c b/src/pager.c index 2c8dceb750..8f49e656e8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6692,6 +6692,14 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } +/* +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. +*/ +sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +} + /* ** Return the full pathname of the journal file. */ diff --git a/src/pager.h b/src/pager.h index ba4eec438d..3552a876e7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,7 @@ int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); +sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index be58c7cd1f..2f61704a2d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1010,6 +1010,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_ZIPVFS 25 #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE diff --git a/src/wal.c b/src/wal.c index 4238ea483b..f38e24a961 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3310,4 +3310,10 @@ int sqlite3WalFramesize(Wal *pWal){ } #endif +/* Return the sqlite3_file object for the WAL file +*/ +sqlite3_file *sqlite3WalFile(Wal *pWal){ + return pWal->pWalFd; +} + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 94a049493e..97e6ab4f10 100644 --- a/src/wal.h +++ b/src/wal.h @@ -44,6 +44,7 @@ # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 +# define sqlite3WalFile(x) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -138,5 +139,8 @@ void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); int sqlite3WalFramesize(Wal *pWal); #endif +/* Return the sqlite3_file object for the WAL file */ +sqlite3_file *sqlite3WalFile(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ From 4ca02079cfc75005f591e1f9b61e2c615896efa7 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jan 2016 07:53:09 +0000 Subject: [PATCH 19/44] Fix fts5vocab.test so that it works with detail=none tables. FossilOrigin-Name: d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 --- ext/fts5/test/fts5vocab.test | 32 +++++++++++++++++++------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index 1f5fa91663..823b2b2872 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -23,8 +23,6 @@ ifcapable !fts5 { foreach_detail_mode $testprefix { -if {[detail_is_none]} continue - proc null_list_entries {iFirst nInterval L} { for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { lset L $i {} @@ -32,13 +30,6 @@ proc null_list_entries {iFirst nInterval L} { return $L } -proc null_insert {iFirst nInterval L} { - for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { - lset L $i {} - } - return $L -} - proc star_from_row {L} { if {[detail_is_full]==0} { set L [null_list_entries 2 3 $L] @@ -57,6 +48,15 @@ proc star_from_col {L} { return $L } +proc row_to_col {L} { + if {[detail_is_none]==0} { error "this is for detail=none mode" } + set ret [list] + foreach {a b c} $L { + lappend ret $a {} $b {} + } + set ret +} + do_execsql_test 1.1.1 { CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); @@ -140,6 +140,9 @@ set res_col [star_from_col { g a 5 7 g b 5 7 x a 1 1 y b 1 1 }] +if {[detail_is_none]} { + set res_col [row_to_col $res_row] +} foreach {tn tbl resname} { 1 "fts5vocab(tt, 'col')" res_col @@ -314,6 +317,7 @@ foreach {term} { set resc [star_from_col $resc] set resr [star_from_row $resr] + if {[detail_is_none]} { set resc [row_to_col $resr] } do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } @@ -378,10 +382,12 @@ do_execsql_test 7.3.1 { SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; } {30} -do_execsql_test 7.3.2 { - SELECT count(*) FROM txc, txc_c - WHERE txc.term = txc_c.term AND txc.col=txc_c.col; -} {57} +if {![detail_is_none]} { + do_execsql_test 7.3.2 { + SELECT count(*) FROM txc, txc_c + WHERE txc.term = txc_c.term AND txc.col=txc_c.col; + } {57} +} } diff --git a/manifest b/manifest index d860431936..459ee65528 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sfts5vocab\smodule\sto\swork\swith\sdetail=col\sand\sdetail=none\stables. -D 2016-01-07T20:07:41.638 +C Fix\sfts5vocab.test\sso\sthat\sit\sworks\swith\sdetail=none\stables. +D 2016-01-08T07:53:09.125 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -179,7 +179,7 @@ F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test f4ff3ba4036a83678f2a8ba8b89eb303bebc122e +F ext/fts5/test/fts5vocab.test e4b12f238f113795615ba6343b63fb326d6a360e F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -1409,7 +1409,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 625695b3d7d4eae7536b5f761b96cd7519ac6ada -R 276342437a8f737c3f32b7f1a4e61d27 +P eedd095dc1c81ce45df00093ba237dd7b3cdff3d +R bd0f99ecb522e0859c1abeb3e9044459 U dan -Z 8c87f6adf59add172dd12da497a3ce1f +Z 0717ee91389ddd999cbabbcc60656596 diff --git a/manifest.uuid b/manifest.uuid index 46d1607a71..042f8c6f9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eedd095dc1c81ce45df00093ba237dd7b3cdff3d \ No newline at end of file +d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 \ No newline at end of file From 4f40cc6f3dc8f2dca8d7020238b22786a3a541b9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jan 2016 17:21:18 +0000 Subject: [PATCH 20/44] Begin adding fts5 tests involving synonyms and detail=none/col tables. FossilOrigin-Name: b3e6f15ec2d9a834e2c80b91ffd7097553816228 --- ext/fts5/test/fts5_common.tcl | 154 ++++++++++++++++++++++---------- ext/fts5/test/fts5synonym2.test | 139 ++++++++++++++++++++++++++++ manifest | 13 +-- manifest.uuid | 2 +- 4 files changed, 252 insertions(+), 56 deletions(-) create mode 100644 ext/fts5/test/fts5synonym2.test diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index aadf148716..2c7fedcefe 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -202,15 +202,24 @@ proc fts5_rnddoc {n} { # -near N (NEAR distance. Default 10) # -col C (List of column indexes to match against) # -pc VARNAME (variable in caller frame to use for phrase numbering) +# -dict VARNAME (array in caller frame to use for synonyms) # proc nearset {aCol args} { + + # Process the command line options. + # set O(-near) 10 set O(-col) {} set O(-pc) "" + set O(-dict) "" set nOpt [lsearch -exact $args --] if {$nOpt<0} { error "no -- option" } + # Set $lPhrase to be a list of phrases. $nPhrase its length. + set lPhrase [lrange $args [expr $nOpt+1] end] + set nPhrase [llength $lPhrase] + foreach {k v} [lrange $args 0 [expr $nOpt-1]] { if {[info exists O($k)]==0} { error "unrecognized option $k" } set O($k) $v @@ -222,9 +231,7 @@ proc nearset {aCol args} { upvar $O(-pc) counter } - # Set $phraselist to be a list of phrases. $nPhrase its length. - set phraselist [lrange $args [expr $nOpt+1] end] - set nPhrase [llength $phraselist] + if {$O(-dict)!=""} { upvar $O(-dict) aDict } for {set j 0} {$j < [llength $aCol]} {incr j} { for {set i 0} {$i < $nPhrase} {incr i} { @@ -232,41 +239,54 @@ proc nearset {aCol args} { } } - set iCol -1 - foreach col $aCol { - incr iCol - if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue - set nToken [llength $col] + # Loop through each column of the current row. + for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { - set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] - for { } {$iFL < $nToken} {incr iFL} { - for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { - set B($iPhrase) [list] - } + # If there is a column filter, test whether this column is excluded. If + # so, skip to the next iteration of this loop. Otherwise, set zCol to the + # column value and nToken to the number of tokens that comprise it. + if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue + set zCol [lindex $aCol $iCol] + set nToken [llength $zCol] + + # Each iteration of the following loop searches a substring of the + # column value for phrase matches. The last token of the substring + # is token $iLast of the column value. The first token is: + # + # iFirst = ($iLast - $O(-near) - 1) + # + # where $sz is the length of the phrase being searched for. A phrase + # counts as matching the substring if its first token lies on or before + # $iLast and its last token on or after $iFirst. + # + # For example, if the query is "NEAR(a+b c, 2)" and the column value: + # + # "x x x x A B x x C x" + # 0 1 2 3 4 5 6 7 8 9" + # + # when (iLast==8 && iFirst=5) the range will contain both phrases and + # so both instances can be added to the output poslists. + # + set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] + for { } {$iLast < $nToken} {incr iLast} { + + catch { array unset B } for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { - set p [lindex $phraselist $iPhrase] + set p [lindex $lPhrase $iPhrase] set nPm1 [expr {[llength $p] - 1}] - set iFirst [expr $iFL - $O(-near) - [llength $p]] - - for {set i $iFirst} {$i <= $iFL} {incr i} { - set lCand [lrange $col $i [expr $i+$nPm1]] + set iFirst [expr $iLast - $O(-near) - [llength $p]] + for {set i $iFirst} {$i <= $iLast} {incr i} { + set lCand [lrange $zCol $i [expr $i+$nPm1]] set bMatch 1 foreach tok $p term $lCand { - if {[string match $tok $term]==0} { - #puts "$tok $term failed" - set bMatch 0 - } + if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break } } - if {$bMatch} { - #puts "match at $i" - lappend B($iPhrase) $i - } - - #if {$lCand == $p} { lappend B($iPhrase) $i } + if {$bMatch} { lappend B($iPhrase) $i } } - if {[llength $B($iPhrase)] == 0} break + + if {![info exists B($iPhrase)]} break } if {$iPhrase==$nPhrase} { @@ -294,6 +314,18 @@ proc nearset {aCol args} { sort_poslist $res } +proc nearset_match {aDictVar tok term} { + if {[string match $tok $term]} { return 1 } + + upvar $aDictVar aDict + if {[info exists aDict($tok)]} { + foreach s $aDict($tok) { + if {[string match $s $term]} { return 1 } + } + } + return 0; +} + #------------------------------------------------------------------------- # Usage: # @@ -405,7 +437,6 @@ proc fts5_poslist2collist {poslist} { } # Comparison function used by fts5_poslist2collist to sort collist entries. -# proc fts5_collist_elem_compare {a b} { foreach {a1 a2} [split $a .] {} foreach {b1 b2} [split $b .] {} @@ -426,17 +457,23 @@ proc fts5_collist_elem_compare {a b} { # FROM $tbl('$expr') # ORDER BY rowid $order; # -proc fts5_query_data {expr tbl {order ASC}} { +proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} { # Figure out the set of columns in the FTS5 table. This routine does # not handle tables with UNINDEXED columns, but if it did, it would # have to be here. db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } + set d "" + if {$aDictVar != ""} { + upvar $aDictVar aDict + set d aDict + } + set cols "" foreach e $lCols { append cols ", '$e'" } set tclexpr [db one [subst -novar { - SELECT fts5_expr_tcl( $expr, 'nearset $cols -pc ::pc' [set cols] ) + SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] ) }]] set res [list] @@ -457,9 +494,17 @@ proc fts5_query_data {expr tbl {order ASC}} { #------------------------------------------------------------------------- # Similar to [fts5_query_data], but omit the collist field. # -proc fts5_poslist_data {expr tbl {order ASC}} { +proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} { set res [list] - foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order] { + + if {$aDictVar!=""} { + upvar $aDictVar aDict + set dict aDict + } else { + set dict "" + } + + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { lappend res $rowid $poslist } set res @@ -467,22 +512,15 @@ proc fts5_poslist_data {expr tbl {order ASC}} { #------------------------------------------------------------------------- # -proc nearset_rf {aCol args} { - set idx [lsearch -exact $args --] - if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { - set ::expr_not_ok 1 - } - list -} - -proc nearset_rc {aCol args} { - nearset_rf $aCol {*}$args - if {[lsearch $args -col]>=0} { - set ::expr_not_ok 1 - } - list -} +# This command will only work inside a [foreach_detail_mode] block. It tests +# whether or not expression $expr run on FTS5 table $tbl is supported by +# the current mode. If so, 1 is returned. If not, 0. +# +# detail=full (all queries supported) +# detail=col (all but phrase queries and NEAR queries) +# detail=none (all but phrase queries, NEAR queries, and column filters) +# proc fts5_expr_ok {expr tbl} { if {![detail_is_full]} { @@ -505,3 +543,21 @@ proc fts5_expr_ok {expr tbl} { return 1 } +# Helper for [fts5_expr_ok] +proc nearset_rf {aCol args} { + set idx [lsearch -exact $args --] + if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { + set ::expr_not_ok 1 + } + list +} + +# Helper for [fts5_expr_ok] +proc nearset_rc {aCol args} { + nearset_rf $aCol {*}$args + if {[lsearch $args -col]>=0} { + set ::expr_not_ok 1 + } + list +} + diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test new file mode 100644 index 0000000000..e3a8003758 --- /dev/null +++ b/ext/fts5/test/fts5synonym2.test @@ -0,0 +1,139 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Code for a simple Tcl tokenizer that supports synonyms at query time. +# +foreach SYNDICT { + {zero 0} + {one 1 i} + {two 2 ii} + {three 3 iii} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} +} { + foreach s $SYNDICT { + set o [list] + foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags == "query"} { + foreach s $::SYNDICT($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + } + } +} + +proc tcl_create {args} { + return "tcl_tokenize" +} + +# +# End of tokenizer code. +#------------------------------------------------------------------------- + +foreach_detail_mode $testprefix { + +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); + + INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); + INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); + INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); + INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); + INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); + INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4'); + INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4'); + INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii'); + INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five'); + INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine'); + INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven'); + INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three'); + INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9'); + INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight'); + INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven'); + INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four'); + INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2'); + INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six'); + INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi'); + INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi'); + INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii'); + INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii'); + INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4'); + INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii'); + INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4'); + INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii'); + INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four'); + INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five'); + INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one'); + INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8'); + INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i'); + INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9'); + INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five'); + INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6'); + INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4'); + INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv'); + INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine'); + INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five'); + INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi'); + INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii'); + INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two'); + INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one'); + INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii'); + INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4'); + INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii'); + INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii'); + INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4'); + INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii'); + INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one'); +} + +foreach {tn expr} { + 1 "eight" +} { + if {[fts5_expr_ok $expr ss]==0} { + do_test 1.$tn.OMITTED { list } [list] + continue + } + + set res [fts5_query_data $expr ss ASC ::SYNDICT] + do_execsql_test 1.$tn.[llength $res].asc { + SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) + } $res +} + +} + +finish_test + diff --git a/manifest b/manifest index 459ee65528..48daf058c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfts5vocab.test\sso\sthat\sit\sworks\swith\sdetail=none\stables. -D 2016-01-08T07:53:09.125 +C Begin\sadding\sfts5\stests\sinvolving\ssynonyms\sand\sdetail=none/col\stables. +D 2016-01-08T17:21:18.901 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -114,7 +114,7 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 +F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced @@ -173,6 +173,7 @@ F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 +F ext/fts5/test/fts5synonym2.test 6aa842d0e5bd019db0c3597e0860eb68eb2867e4 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1409,7 +1410,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eedd095dc1c81ce45df00093ba237dd7b3cdff3d -R bd0f99ecb522e0859c1abeb3e9044459 +P d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 +R c443735bd0d019a3a4798f3f3350cb78 U dan -Z 0717ee91389ddd999cbabbcc60656596 +Z 48c62f9fa751846d5897bddd7cc5158f diff --git a/manifest.uuid b/manifest.uuid index 042f8c6f9c..dc0f491a48 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 \ No newline at end of file +b3e6f15ec2d9a834e2c80b91ffd7097553816228 \ No newline at end of file From 50922cfc7c12514d44bc6641e9e44833eb4b637b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Jan 2016 14:19:14 +0000 Subject: [PATCH 21/44] Performance optimization in sqlite3WalkExpr(). FossilOrigin-Name: 2f8bd5fab8cc51d1f8932c3490fd24dbccf6ef31 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/walker.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b94535791e..7c3ae97b20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.11.0\sdue\sto\sthe\sWAL\soverwrite\senhancement. -D 2016-01-11T12:52:55.912 +C Performance\soptimization\sin\ssqlite3WalkExpr(). +D 2016-01-11T14:19:14.339 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -411,7 +411,7 @@ F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 92ca9e7923c337c497e4c9aa7edac800e269a1d8 F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b -F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba +F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 F src/where.c c6d3d2f6af57d574a7365ee2b225a5024f2a6bec F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a F src/wherecode.c dfbfe198e418b01f208b489e088edd230c91a4e7 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d493d4f1186c15da50558a13490cba0191e6f649 -R a92483f04fb0661e2d9ab2766c996317 +P 8e807bfaa197027d0cb73532baa96755ce71ea12 +R 177e5551beb5399a29df920e18a04961 U drh -Z d2a7370395d09ae9d2ddb7bbbb25edc7 +Z b8d12fcc18213355136dacd10d5cf924 diff --git a/manifest.uuid b/manifest.uuid index 7d0842e5bd..1f83d937d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e807bfaa197027d0cb73532baa96755ce71ea12 \ No newline at end of file +2f8bd5fab8cc51d1f8932c3490fd24dbccf6ef31 \ No newline at end of file diff --git a/src/walker.c b/src/walker.c index 81e0f2cd60..1e0ad32871 100644 --- a/src/walker.c +++ b/src/walker.c @@ -36,9 +36,8 @@ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; - if( pExpr==0 ) return WRC_Continue; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); rc = pWalker->xExprCallback(pWalker, pExpr); @@ -54,6 +53,9 @@ int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ } return rc & WRC_Abort; } +int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; +} /* ** Call sqlite3WalkExpr() for every expression in list p or until From aaf17c41ec558e2c0c174055c6dcbbd1908aa290 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Jan 2016 17:30:28 +0000 Subject: [PATCH 22/44] Fix bugs in fts5 synonym processing for detail=col and other modes. FossilOrigin-Name: 0e3c545423246231ddac8fed2f103d71d556e17f --- ext/fts5/fts5_expr.c | 106 +++++++++++++++++++++++--------- ext/fts5/test/fts5synonym2.test | 25 ++++++-- manifest | 14 ++--- manifest.uuid | 2 +- 4 files changed, 106 insertions(+), 41 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 3804a2abfe..16209bf4ba 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*); struct Fts5Expr { Fts5Index *pIndex; + Fts5Config *pConfig; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ @@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew( }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; + pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; @@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ /* ** Argument pTerm must be a synonym iterator. */ -static int fts5ExprSynonymPoslist( +static int fts5ExprSynonymList( Fts5ExprTerm *pTerm, + int bCollist, Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ @@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist( if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; - i64 dummy; - rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + + if( bCollist ){ + rc = sqlite3Fts5IterCollist(pIter, &a, &n); + }else{ + i64 dummy; + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + } + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( n==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); @@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch( int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist( - pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + rc = fts5ExprSynonymList( + pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); @@ -757,30 +767,51 @@ static int fts5ExprNearTest( ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; - int i; - /* Check that each phrase in the nearset matches the current row. - ** Populate the pPhrase->poslist buffers at the same time. If any - ** phrase is not a match, break out of the loop early. */ - for(i=0; rc==SQLITE_OK && inPhrase; i++){ - Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ - int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); - if( bMatch==0 ) break; - }else{ - rc = sqlite3Fts5IterPoslistBuffer( - pPhrase->aTerm[0].pIter, &pPhrase->poslist - ); + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + int n; + i64 iRowid; + rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + }else if( iRowid==pNode->iRowid && n>0 ){ + pPhrase->poslist.n = 1; + } + } } - } + return pPhrase->poslist.n; + }else{ + int i; - *pRc = rc; - if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ - return 1; - } + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + rc = sqlite3Fts5IterPoslistBuffer( + pPhrase->aTerm[0].pIter, &pPhrase->poslist + ); + } + } - return 0; + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } } static int fts5ExprTokenTest( @@ -1606,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase( if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; @@ -2453,17 +2485,35 @@ int sqlite3Fts5ExprPhraseCollist( ){ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; Fts5ExprNode *pNode = pPhrase->pNode; - assert( iPhrase>=0 && iPhrasenPhrase ); + int rc = SQLITE_OK; + assert( iPhrase>=0 && iPhrasenPhrase ); if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 ){ - sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + int bDel = 0; + u8 *a; + rc = fts5ExprSynonymList( + pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist + ); + if( bDel ){ + sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); + *ppCollist = pPhrase->poslist.p; + sqlite3_free(a); + }else{ + *ppCollist = a; + } + }else{ + sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + } }else{ *ppCollist = 0; *pnCollist = 0; } - return SQLITE_OK; + + return rc; } diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index e3a8003758..96cacb293f 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -13,7 +13,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5synonym +set testprefix fts5synonym2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { @@ -47,7 +47,7 @@ proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags == "query"} { - foreach s $::SYNDICT($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } } } } @@ -67,7 +67,6 @@ fts5_aux_test_functions db do_execsql_test 1.0 { CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); - INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); @@ -120,14 +119,30 @@ do_execsql_test 1.0 { } foreach {tn expr} { - 1 "eight" + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" + 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" + 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" + + 2.1 "one OR two OR three OR four" + 2.2 "(one AND two) OR (three AND four)" + 2.3 "(one AND two) OR (three AND four) NOT five" + 2.4 "(one AND two) NOT 6" + + 3.1 "b:one AND a:two" + 3.2 "b:one OR a:two" + 3.3 "a:one OR b:1 OR {a b} : i" + + 4.1 "NEAR(one two, 2)" + 4.2 "NEAR(one two three, 2)" + 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" } { if {[fts5_expr_ok $expr ss]==0} { do_test 1.$tn.OMITTED { list } [list] continue } - set res [fts5_query_data $expr ss ASC ::SYNDICT] + set res [fts5_query_data $expr ss ASC ::syn] + breakpoint do_execsql_test 1.$tn.[llength $res].asc { SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) } $res diff --git a/manifest b/manifest index 48daf058c3..e4ef860b2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sadding\sfts5\stests\sinvolving\ssynonyms\sand\sdetail=none/col\stables. -D 2016-01-08T17:21:18.901 +C Fix\sbugs\sin\sfts5\ssynonym\sprocessing\sfor\sdetail=col\sand\sother\smodes. +D 2016-01-11T17:30:28.056 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -101,7 +101,7 @@ F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 913ab9f9a7651d76f055f9847ca0ae0e2f5da78d +F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1 F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e @@ -173,7 +173,7 @@ F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 -F ext/fts5/test/fts5synonym2.test 6aa842d0e5bd019db0c3597e0860eb68eb2867e4 +F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1410,7 +1410,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 -R c443735bd0d019a3a4798f3f3350cb78 +P b3e6f15ec2d9a834e2c80b91ffd7097553816228 +R f239c4ba67c1753046748953bb2cde92 U dan -Z 48c62f9fa751846d5897bddd7cc5158f +Z 915aa1f2ec9df261dd260c37cdbace05 diff --git a/manifest.uuid b/manifest.uuid index dc0f491a48..3b1a59f4b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3e6f15ec2d9a834e2c80b91ffd7097553816228 \ No newline at end of file +0e3c545423246231ddac8fed2f103d71d556e17f \ No newline at end of file From 69f6e2587bef21d7faa3f11748529ac8b96f8641 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Jan 2016 18:05:00 +0000 Subject: [PATCH 23/44] Performance optimization on the Column opcode. FossilOrigin-Name: ecc98bef43c2fd07d64e4efddf340929a875ebac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 54 ++++++++++++++++++++++++++------------------------- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 7c3ae97b20..e4dfdc13fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\ssqlite3WalkExpr(). -D 2016-01-11T14:19:14.339 +C Performance\soptimization\son\sthe\sColumn\sopcode. +D 2016-01-11T18:05:00.482 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -398,7 +398,7 @@ F src/update.c 17332f9fe818cbc0444c36a811800af8498af4c3 F src/utf.c 32d7f82aa921322f3e1c956f4b58f019ebd2c6b3 F src/util.c e802e8e311a0d6c48cd1b3e89db164f6f0248d70 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c 6ac8e5d808d48afc369316e147c191102f0584c1 +F src/vdbe.c 6572d00eefeaa0b14b325fdf3a409920ec3fee82 F src/vdbe.h efb7a8c1459e31f3ea4377824c6a7e4cb5068637 F src/vdbeInt.h 42eefa4f9e7432b9968d321b44e48821ec13b189 F src/vdbeapi.c 020681b943e77766b32ae1cddf86d7831b7374ca @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e807bfaa197027d0cb73532baa96755ce71ea12 -R 177e5551beb5399a29df920e18a04961 +P 2f8bd5fab8cc51d1f8932c3490fd24dbccf6ef31 +R 68508ce4367c0efdf38feafa98972c4a U drh -Z b8d12fcc18213355136dacd10d5cf924 +Z e90441a49428aea3c83069305613bb8e diff --git a/manifest.uuid b/manifest.uuid index 1f83d937d4..bccf8bb6b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f8bd5fab8cc51d1f8932c3490fd24dbccf6ef31 \ No newline at end of file +ecc98bef43c2fd07d64e4efddf340929a875ebac \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5acb3b3137..7eb4c0baeb 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2374,7 +2374,6 @@ case OP_Column: { u64 offset64; /* 64-bit offset */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ - u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ p2 = pOp->p2; @@ -2552,10 +2551,31 @@ case OP_Column: { assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); assert( t==pC->aType[p2] ); + pDest->enc = encoding; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); + zData = pC->aRow + aOffset[p2]; + if( t<12 ){ + sqlite3VdbeSerialGet(zData, t, pDest); + }else{ + /* If the column value is a string, we need a persistent value, not + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). + */ + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; + pDest->n = len = (t-12)/2; + if( pDest->szMalloc < len+2 ){ + pDest->flags = MEM_Null; + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; + }else{ + pDest->z = pDest->zMalloc; + } + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = aFlag[t&1]; + } }else{ /* This branch happens only when content is on overflow pages */ if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 @@ -2567,38 +2587,20 @@ case OP_Column: { ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. NULL will work for the value for strings - ** and blobs and whatever is in the payloadSize64 variable - ** will work for everything else. */ - sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); + ** content from disk. */ + static u8 aZero[8]; /* This is the bogus content */ + sqlite3VdbeSerialGet(aZero, t, pDest); }else{ rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, pDest); - if( rc!=SQLITE_OK ){ - goto op_column_error; + if( rc==SQLITE_OK ){ + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; } - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - pDest->flags &= ~MEM_Ephem; } } - pDest->enc = encoding; op_column_out: - /* If the column value is an ephemeral string, go ahead and persist - ** that string in case the cursor moves before the column value is - ** used. The following code does the equivalent of Deephemeralize() - ** but does it faster. */ - if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ - fx = pDest->flags & (MEM_Str|MEM_Blob); - assert( fx!=0 ); - zData = (const u8*)pDest->z; - len = pDest->n; - if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; - memcpy(pDest->z, zData, len); - pDest->z[len] = 0; - pDest->z[len+1] = 0; - pDest->flags = fx|MEM_Term; - } op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); From 6a657b2477c4b8d5b67506e9a553a150d482bc1b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Jan 2016 18:23:07 +0000 Subject: [PATCH 24/44] Add the fts5speed.tcl script to ext/fts5/tool/. FossilOrigin-Name: 2c0b93553ebb00a70c611eb8dac021877933f088 --- ext/fts5/tool/fts5speed.tcl | 59 +++++++++++++++++++++++++++++++++++++ manifest | 11 +++---- manifest.uuid | 2 +- 3 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 ext/fts5/tool/fts5speed.tcl diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl new file mode 100644 index 0000000000..1b060ea759 --- /dev/null +++ b/ext/fts5/tool/fts5speed.tcl @@ -0,0 +1,59 @@ + + +set Q { + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"} + {25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"} + {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"} + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"} + + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} +} + +proc usage {} { + global Q + puts stderr "Usage: $::argv0 DATABASE QUERY" + puts stderr "" + for {set i 1} {$i <= [llength $Q]} {incr i} { + puts stderr " $i. [lindex $Q [expr $i-1]]" + } + puts stderr "" + exit -1 +} + + +set nArg [llength $argv] +if {$nArg!=2 && $nArg!=3} usage +set database [lindex $argv 0] +set iquery [lindex $argv 1] +if {$iquery<1 || $iquery>[llength $Q]} usage +set nRepeat 0 +if {$nArg==3} { set nRepeat [lindex $argv 2] } + + +sqlite3 db $database +catch { load_static_extension db fts5 } + +incr iquery -1 +set sql [lindex $Q $iquery 1] +if {$nRepeat==0} { + set nRepeat [lindex $Q $iquery 0] +} + +puts "sql: $sql" +puts "nRepeat: $nRepeat" +if {[regexp matchinfo $sql]} { + sqlite3_fts5_register_matchinfo db + db eval $sql +} else { + puts "result: [db eval $sql]" +} + +for {set i 1} {$i < $nRepeat} {incr i} { + db eval $sql +} + + diff --git a/manifest b/manifest index e4ef860b2c..6158acb724 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbugs\sin\sfts5\ssynonym\sprocessing\sfor\sdetail=col\sand\sother\smodes. -D 2016-01-11T17:30:28.056 +C Add\sthe\sfts5speed.tcl\sscript\sto\sext/fts5/tool/. +D 2016-01-11T18:23:07.397 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -181,6 +181,7 @@ F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test e4b12f238f113795615ba6343b63fb326d6a360e +F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221 F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -1410,7 +1411,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3e6f15ec2d9a834e2c80b91ffd7097553816228 -R f239c4ba67c1753046748953bb2cde92 +P 0e3c545423246231ddac8fed2f103d71d556e17f +R af2ae430a57fbc5161db0dbbd31b8a3f U dan -Z 915aa1f2ec9df261dd260c37cdbace05 +Z 7e47e5a1c7084b8fa203746692b6702a diff --git a/manifest.uuid b/manifest.uuid index 3b1a59f4b6..7a083e6ea4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e3c545423246231ddac8fed2f103d71d556e17f \ No newline at end of file +2c0b93553ebb00a70c611eb8dac021877933f088 \ No newline at end of file From 46414c29ba18e587643f8e95c9232367ef0839d0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 11 Jan 2016 22:43:04 +0000 Subject: [PATCH 25/44] Fix duplicate test number in 'waloverwrite'. FossilOrigin-Name: f5fb82686697e5fa1937323b1c1b35cf90ad0302 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/waloverwrite.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e4dfdc13fc..6a59c023df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\son\sthe\sColumn\sopcode. -D 2016-01-11T18:05:00.482 +C Fix\sduplicate\stest\snumber\sin\s'waloverwrite'. +D 2016-01-11T22:43:04.382 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -1301,7 +1301,7 @@ F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 -F test/waloverwrite.test 8702964967c2f28204f0b4f27af65da81999fc99 +F test/waloverwrite.test a0d2ae0783187374c1e6a9571e0916152977cb81 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2f8bd5fab8cc51d1f8932c3490fd24dbccf6ef31 -R 68508ce4367c0efdf38feafa98972c4a -U drh -Z e90441a49428aea3c83069305613bb8e +P ecc98bef43c2fd07d64e4efddf340929a875ebac +R bffff77db2267e7f44f05491f1b6c72d +U mistachkin +Z 70102cc7ed23bad282b82f2981075253 diff --git a/manifest.uuid b/manifest.uuid index bccf8bb6b0..d694eba5b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecc98bef43c2fd07d64e4efddf340929a875ebac \ No newline at end of file +f5fb82686697e5fa1937323b1c1b35cf90ad0302 \ No newline at end of file diff --git a/test/waloverwrite.test b/test/waloverwrite.test index 5e51a70d3f..aa1154fb97 100644 --- a/test/waloverwrite.test +++ b/test/waloverwrite.test @@ -154,7 +154,7 @@ foreach {tn xtra} { execsql { SELECT sum(length(y)) FROM t1 } db2 } [expr 20*798] - do_test 1.$tn.9 { + do_test 1.$tn.10 { execsql { PRAGMA integrity_check } db2 } ok db2 close From 00dcecab193d4b9c008caa50ccb10f479a5aae96 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Jan 2016 22:58:50 +0000 Subject: [PATCH 26/44] Performance optimization in the sqlite3VdbeChangeP4() routine of the code generator. FossilOrigin-Name: 28bd8d5fc541464b69886b7e886939035b42a869 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 68 +++++++++++++++++++++++---------------------------- src/where.c | 3 +-- 4 files changed, 41 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 6a59c023df..41cffbc0d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sduplicate\stest\snumber\sin\s'waloverwrite'. -D 2016-01-11T22:43:04.382 +C Performance\soptimization\sin\sthe\ssqlite3VdbeChangeP4()\sroutine\sof\sthe\ncode\sgenerator. +D 2016-01-11T22:58:50.164 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -402,7 +402,7 @@ F src/vdbe.c 6572d00eefeaa0b14b325fdf3a409920ec3fee82 F src/vdbe.h efb7a8c1459e31f3ea4377824c6a7e4cb5068637 F src/vdbeInt.h 42eefa4f9e7432b9968d321b44e48821ec13b189 F src/vdbeapi.c 020681b943e77766b32ae1cddf86d7831b7374ca -F src/vdbeaux.c 5d8c7c04e0f677033efb1292248a5f9056bbc34c +F src/vdbeaux.c f2e6b4fae037db04323be8de7bcd266375746471 F src/vdbeblob.c fdc4a81605ae7a35ae94a55bd768b66d6be16f15 F src/vdbemem.c fdd1578e47bea61390d472de53c565781d81e045 F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d @@ -412,7 +412,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 92ca9e7923c337c497e4c9aa7edac800e269a1d8 F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c c6d3d2f6af57d574a7365ee2b225a5024f2a6bec +F src/where.c bb69654f841ae7af0a20cc6fb8f0ac57901c31be F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a F src/wherecode.c dfbfe198e418b01f208b489e088edd230c91a4e7 F src/whereexpr.c eebba8340c90de73b3d3bbe8c43b84559b8e6e2c @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ecc98bef43c2fd07d64e4efddf340929a875ebac -R bffff77db2267e7f44f05491f1b6c72d -U mistachkin -Z 70102cc7ed23bad282b82f2981075253 +P f5fb82686697e5fa1937323b1c1b35cf90ad0302 +R 5cc83036f9c9da1fadf85dd4abd0f133 +U drh +Z a8c9a7d6a102c57d4a0aed7eac3aa290 diff --git a/manifest.uuid b/manifest.uuid index d694eba5b3..aa9453cfcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5fb82686697e5fa1937323b1c1b35cf90ad0302 \ No newline at end of file +28bd8d5fc541464b69886b7e886939035b42a869 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index aa8070a132..2f65d85777 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -305,8 +305,7 @@ int sqlite3VdbeAddOp4Dup8( */ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ int j; - int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); - sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); } @@ -805,7 +804,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - freeP4(db, pOp->p4type, pOp->p4.p); + if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif @@ -867,16 +866,34 @@ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** ** If addr<0 then change P4 on the most recently inserted instruction. */ +static void SQLITE_NOINLINE vdbeChangeP4Full( + Vdbe *p, + Op *pOp, + const char *zP4, + int n +){ + if( pOp->p4type ){ + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = 0; + pOp->p4.p = 0; + } + if( n<0 ){ + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ Op *pOp; sqlite3 *db; assert( p!=0 ); db = p->db; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->aOp==0 || db->mallocFailed ){ - if( n!=P4_VTAB ){ - freeP4(db, n, (void*)*(char**)&zP4); - } + assert( p->aOp!=0 || db->mallocFailed ); + if( db->mallocFailed ){ + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); return; } assert( p->nOp>0 ); @@ -885,43 +902,20 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED - || pOp->p4type==P4_INT32 - || pOp->p4type==P4_KEYINFO ); - freeP4(db, pOp->p4type, pOp->p4.p); - pOp->p4.p = 0; + if( n>=0 || pOp->p4type ){ + vdbeChangeP4Full(p, pOp, zP4, n); + return; + } if( n==P4_INT32 ){ /* Note: this cast is safe, because the origin data point was an int ** that was cast to a (const char *). */ pOp->p4.i = SQLITE_PTR_TO_INT(zP4); pOp->p4type = P4_INT32; - }else if( zP4==0 ){ - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; - }else if( n==P4_KEYINFO ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_KEYINFO; -#ifdef SQLITE_ENABLE_CURSOR_HINTS - }else if( n==P4_EXPR ){ - /* Responsibility for deleting the Expr tree is handed over to the - ** VDBE by this operation. The caller should have already invoked - ** sqlite3ExprDup() or whatever other routine is needed to make a - ** private copy of the tree. */ - pOp->p4.pExpr = (Expr*)zP4; - pOp->p4type = P4_EXPR; -#endif - }else if( n==P4_VTAB ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_VTAB; - sqlite3VtabLock((VTable *)zP4); - assert( ((VTable *)zP4)->db==p->db ); - }else if( n<0 ){ + }else if( zP4!=0 ){ + assert( n<0 ); pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; - }else{ - if( n==0 ) n = sqlite3Strlen30(zP4); - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); - pOp->p4type = P4_DYNAMIC; + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); } } diff --git a/src/where.c b/src/where.c index e86e26ef1a..2cb8334ffb 100644 --- a/src/where.c +++ b/src/where.c @@ -4318,8 +4318,7 @@ WhereInfo *sqlite3WhereBegin( Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, - SQLITE_INT_TO_PTR(n), P4_INT32); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS From e2258a2076c237a70d4dbec7be3754fceebc2f1d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 00:37:55 +0000 Subject: [PATCH 27/44] Various #ifdef enhancements for improved VxWorks support. FossilOrigin-Name: 75cd41ff179e29c5d45f9d7fed784bc339e0d7a0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 16 +++++++++++++--- src/vxworks.h | 2 ++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 41cffbc0d6..7ae248f6b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\ssqlite3VdbeChangeP4()\sroutine\sof\sthe\ncode\sgenerator. -D 2016-01-11T22:58:50.164 +C Various\s#ifdef\senhancements\sfor\simproved\sVxWorks\ssupport. +D 2016-01-12T00:37:55.072 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -317,7 +317,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 82986e1e75782b54da7822dca42d36d974fc2948 +F src/os_unix.c b509b49b40a269e7b75ab511b6e92b2dc9444359 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 58d2593612acb6b542de6715b4af397ea1fa0a35 @@ -408,7 +408,7 @@ F src/vdbemem.c fdd1578e47bea61390d472de53c565781d81e045 F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 -F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb +F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 F src/wal.c 92ca9e7923c337c497e4c9aa7edac800e269a1d8 F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5fb82686697e5fa1937323b1c1b35cf90ad0302 -R 5cc83036f9c9da1fadf85dd4abd0f133 +P 28bd8d5fc541464b69886b7e886939035b42a869 +R a0f920051028c27b520313db4d779ce4 U drh -Z a8c9a7d6a102c57d4a0aed7eac3aa290 +Z 6a1495f4c4d6026f44345482bd14549e diff --git a/manifest.uuid b/manifest.uuid index aa9453cfcd..6f028487c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28bd8d5fc541464b69886b7e886939035b42a869 \ No newline at end of file +75cd41ff179e29c5d45f9d7fed784bc339e0d7a0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a40a866780..5bd6116945 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -430,7 +430,11 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) +#if defined(HAVE_FCHOWN) { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, @@ -464,7 +468,11 @@ static struct unix_syscall { #endif #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) +#if defined(HAVE_READLINK) { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) @@ -477,10 +485,10 @@ static struct unix_syscall { ** we are not running as root. */ static int robustFchown(int fd, uid_t uid, gid_t gid){ -#if OS_VXWORKS - return 0; -#else +#if defined(HAVE_FCHOWN) return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; #endif } @@ -5947,6 +5955,7 @@ static int unixFullPathname( assert( pVfs->mxPathname==MAX_PATHNAME ); UNUSED_PARAMETER(pVfs); +#if defined(HAVE_READLINK) /* Attempt to resolve the path as if it were a symbolic link. If it is ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if ** the identified file is not a symbolic link or does not exist, then @@ -5962,6 +5971,7 @@ static int unixFullPathname( }else{ zOut[nByte] = '\0'; } +#endif /* If buffer zOut[] now contains an absolute path there is nothing more ** to do. If it contains a relative path, do the following: diff --git a/src/vxworks.h b/src/vxworks.h index 45a44453a7..60c41a19b8 100644 --- a/src/vxworks.h +++ b/src/vxworks.h @@ -26,4 +26,6 @@ #else /* This is not VxWorks. */ #define OS_VXWORKS 0 +#define HAVE_FCHOWN 1 +#define HAVE_READLINK 1 #endif /* defined(_WRS_KERNEL) */ From 9ca33fa4c26467330a13d74f1646b27d869ff147 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 02:00:24 +0000 Subject: [PATCH 28/44] Fix an uninitialized field in the Select object when SELECTTRACE is enabled. FossilOrigin-Name: fc62851667d9120f98a2641c87bbc5a2485482b2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7ae248f6b9..6575eb3b4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Various\s#ifdef\senhancements\sfor\simproved\sVxWorks\ssupport. -D 2016-01-12T00:37:55.072 +C Fix\san\suninitialized\sfield\sin\sthe\sSelect\sobject\swhen\sSELECTTRACE\sis\senabled. +D 2016-01-12T02:00:24.480 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -333,7 +333,7 @@ F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c d84c091185bc160c349e8bf460ebd084dbd77e64 +F src/select.c 372234e3554a7917544242563216af760d7d4219 F src/shell.c ed71dc7679e6f087a3f1ea3f9dae4b0fae7209c3 F src/sqlite.h.in 7d87d71b9a4689c51fa092f48f16590ff71558e3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 28bd8d5fc541464b69886b7e886939035b42a869 -R a0f920051028c27b520313db4d779ce4 +P 75cd41ff179e29c5d45f9d7fed784bc339e0d7a0 +R f2bd43aeba2b6092057d74bc062b9600 U drh -Z 6a1495f4c4d6026f44345482bd14549e +Z 0ab9387fbd675bc198faa7ea2440e420 diff --git a/manifest.uuid b/manifest.uuid index 6f028487c4..cb7f85349f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75cd41ff179e29c5d45f9d7fed784bc339e0d7a0 \ No newline at end of file +fc62851667d9120f98a2641c87bbc5a2485482b2 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b1db07d09a..0e7c4595d1 100644 --- a/src/select.c +++ b/src/select.c @@ -124,6 +124,9 @@ Select *sqlite3SelectNew( pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; +#if SELECTTRACE_ENABLED + pNew->zSelName[0] = 0; +#endif pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; From 869aaf0941f440f5ded3e1757da9f24058e76220 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 02:28:19 +0000 Subject: [PATCH 29/44] Make sure all bytes of the WAL file are initialized even if the checksums are not being computed until commit due to WAL overwrite. FossilOrigin-Name: 1f44a9bdc1577aed088d9b91882527228015dcd3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 6575eb3b4e..52e6f9a768 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\suninitialized\sfield\sin\sthe\sSelect\sobject\swhen\sSELECTTRACE\sis\senabled. -D 2016-01-12T02:00:24.480 +C Make\ssure\sall\sbytes\sof\sthe\sWAL\sfile\sare\sinitialized\seven\sif\sthe\schecksums\sare\nnot\sbeing\scomputed\suntil\scommit\sdue\sto\sWAL\soverwrite. +D 2016-01-12T02:28:19.320 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -409,7 +409,7 @@ F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 -F src/wal.c 92ca9e7923c337c497e4c9aa7edac800e269a1d8 +F src/wal.c ec116eb4dd257561b0b910b422eb0fc459c82c28 F src/wal.h 907943dfdef10b583e81906679a347e0ec6f1b1b F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 F src/where.c bb69654f841ae7af0a20cc6fb8f0ac57901c31be @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 75cd41ff179e29c5d45f9d7fed784bc339e0d7a0 -R f2bd43aeba2b6092057d74bc062b9600 +P fc62851667d9120f98a2641c87bbc5a2485482b2 +R 5117171a7e356e05f85dd4c6a03b017d U drh -Z 0ab9387fbd675bc198faa7ea2440e420 +Z 900d53369fb7ca05847989ae4b11e011 diff --git a/manifest.uuid b/manifest.uuid index cb7f85349f..b281737f3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc62851667d9120f98a2641c87bbc5a2485482b2 \ No newline at end of file +1f44a9bdc1577aed088d9b91882527228015dcd3 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index ae6a54849a..9d711198a8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -708,6 +708,8 @@ static void walEncodeFrame( sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); + }else{ + memset(&aFrame[8], 0, 16); } } From 8f3f65ec1cd54c250984f0b0625bbbcdacbbfd1e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 04:08:54 +0000 Subject: [PATCH 30/44] Remove an unused field from the MemPage object. FossilOrigin-Name: 5fc0a4c1f353c4c06fcbc4bce8bbe1897ef49eee --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 -------- src/btreeInt.h | 1 - 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 52e6f9a768..ca04021326 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sall\sbytes\sof\sthe\sWAL\sfile\sare\sinitialized\seven\sif\sthe\schecksums\sare\nnot\sbeing\scomputed\suntil\scommit\sdue\sto\sWAL\soverwrite. -D 2016-01-12T02:28:19.320 +C Remove\san\sunused\sfield\sfrom\sthe\sMemPage\sobject. +D 2016-01-12T04:08:54.028 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -276,9 +276,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc F src/bitvec.c 1a78d450a17c5016710eec900bedfc5729bf9bdf F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 -F src/btree.c 6bd9b3d778a023e2238a81cd0b87b00085220e0e +F src/btree.c 5d93e2477acb99d50a8b045f2e26a0be3d7751fe F src/btree.h 68ef301795e00cdf1d3ab93abc44a43b7fe771e0 -F src/btreeInt.h b5f2651b41808f038dee9282c5dc0232ce6532d3 +F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 F src/build.c 9d497ff4bf3c82cecb520436e0e9963785627583 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc62851667d9120f98a2641c87bbc5a2485482b2 -R 5117171a7e356e05f85dd4c6a03b017d +P 1f44a9bdc1577aed088d9b91882527228015dcd3 +R dcacd681598087d316aa7da66fd3e6ff U drh -Z 900d53369fb7ca05847989ae4b11e011 +Z a106a14306903405633952b5120c8bc2 diff --git a/manifest.uuid b/manifest.uuid index b281737f3f..c61a4ceebd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f44a9bdc1577aed088d9b91882527228015dcd3 \ No newline at end of file +5fc0a4c1f353c4c06fcbc4bce8bbe1897ef49eee \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 77552d5831..b5a1ee3b2c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1079,7 +1079,6 @@ static void btreeParseCellPtrNoPayload( ){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 ); - assert( pPage->noPayload ); assert( pPage->childPtrSize==4 ); #ifndef SQLITE_DEBUG UNUSED_PARAMETER(pPage); @@ -1101,8 +1100,6 @@ static void btreeParseCellPtr( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - assert( pPage->intKeyLeaf || pPage->noPayload ); - assert( pPage->noPayload==0 ); assert( pPage->intKeyLeaf ); assert( pPage->childPtrSize==0 ); pIter = pCell; @@ -1171,7 +1168,6 @@ static void btreeParseCellPtrIndex( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); assert( pPage->intKeyLeaf==0 ); - assert( pPage->noPayload==0 ); pIter = pCell + pPage->childPtrSize; nPayload = *pIter; if( nPayload>=0x80 ){ @@ -1232,7 +1228,6 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif - assert( pPage->noPayload==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -1690,11 +1685,9 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->intKey = 1; if( pPage->leaf ){ pPage->intKeyLeaf = 1; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtr; }else{ pPage->intKeyLeaf = 0; - pPage->noPayload = 1; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; } @@ -1709,7 +1702,6 @@ static int decodeFlags(MemPage *pPage, int flagByte){ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->noPayload = 0; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; diff --git a/src/btreeInt.h b/src/btreeInt.h index a196e18a28..6cd090257c 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -276,7 +276,6 @@ struct MemPage { u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ - u8 noPayload; /* True if internal intKey page (thus w/o data) */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ From d1dd7507e43a2650d6c4a363dd021653ebbd896f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 14:10:05 +0000 Subject: [PATCH 31/44] Fix a harmless use of an uninitialized variable following system errors in the multi-threaded sorter. FossilOrigin-Name: 4bb2c1df105c2d21f5c4c7ef656ff1d6e15f78bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ca04021326..b4c264a04b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\sfield\sfrom\sthe\sMemPage\sobject. -D 2016-01-12T04:08:54.028 +C Fix\sa\sharmless\suse\sof\san\suninitialized\svariable\sfollowing\ssystem\serrors\nin\sthe\smulti-threaded\ssorter. +D 2016-01-12T14:10:05.866 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -405,7 +405,7 @@ F src/vdbeapi.c 020681b943e77766b32ae1cddf86d7831b7374ca F src/vdbeaux.c f2e6b4fae037db04323be8de7bcd266375746471 F src/vdbeblob.c fdc4a81605ae7a35ae94a55bd768b66d6be16f15 F src/vdbemem.c fdd1578e47bea61390d472de53c565781d81e045 -F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d +F src/vdbesort.c 0971557e5d3c289e46f56a52aed2197c13251de7 F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806 F src/vxworks.h 974e7d9a98f602d6310d563e1dc4e08f9fc48e47 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1f44a9bdc1577aed088d9b91882527228015dcd3 -R dcacd681598087d316aa7da66fd3e6ff +P 5fc0a4c1f353c4c06fcbc4bce8bbe1897ef49eee +R 92b34da2a8b41bb1db044333a3c02410 U drh -Z a106a14306903405633952b5120c8bc2 +Z e6a278c031968913cb3469aed14f2f8a diff --git a/manifest.uuid b/manifest.uuid index c61a4ceebd..9e4a59f983 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5fc0a4c1f353c4c06fcbc4bce8bbe1897ef49eee \ No newline at end of file +4bb2c1df105c2d21f5c4c7ef656ff1d6e15f78bc \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 54e538fd50..e095f80912 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -737,7 +737,7 @@ static int vdbePmaReaderInit( rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ + u64 nByte = 0; /* Size of PMA in bytes */ rc = vdbePmaReadVarint(pReadr, &nByte); pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; From fd7601a125dc1e16f7220540104d9de2be29474e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Jan 2016 19:28:51 +0000 Subject: [PATCH 32/44] Improve performance of fts5 low level iterators. FossilOrigin-Name: bc5118f40a11f64ffb4e1c086277fa80b9764745 --- ext/fts5/fts5_index.c | 376 ++++++++++++++++++++++++-------------- ext/fts5/test/fts5ad.test | 2 - manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 245 insertions(+), 149 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ef673085c6..2488cd7ca9 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -433,6 +433,9 @@ struct Fts5SegIter { Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ + /* Next method */ + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); + /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; @@ -453,7 +456,6 @@ struct Fts5SegIter { i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ u8 bDel; /* True if the delete flag is set */ - // u8 bContent; /* True if has content (detail=none mode) */ }; /* @@ -1575,6 +1577,20 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ fts5SegIterLoadRowid(p, pIter); } +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); + +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + pIter->xNext = fts5SegIterNext_Reverse; + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xNext = fts5SegIterNext_None; + }else{ + pIter->xNext = fts5SegIterNext; + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -1600,6 +1616,7 @@ static void fts5SegIterInit( if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); + fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); @@ -1740,6 +1757,110 @@ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used by reverse iterators. +*/ +static void fts5SegIterNext_Reverse( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + assert( pIter->flags & FTS5_SEGITER_REVERSE ); + assert( pIter->pNextLeaf==0 ); + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + int nPos; + int bDummy; + i64 iDelta; + + pIter->iRowidOffset--; + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; + fts5SegIterLoadNPos(p, pIter); + iOff = pIter->iLeafOffset; + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ + iOff += pIter->nPos; + } + fts5GetVarint(&a[iOff], (u64*)&iDelta); + pIter->iRowid -= iDelta; + }else{ + fts5SegIterReverseNewPage(p, pIter); + } +} + +/* +** Advance iterator pIter to the next entry. +** +** This version of fts5SegIterNext() is only used if detail=none and the +** iterator is not a reverse direction iterator. +*/ +static void fts5SegIterNext_None( + Fts5Index *p, /* FTS5 backend object */ + Fts5SegIter *pIter, /* Iterator to advance */ + int *pbNewTerm /* OUT: Set for new term */ +){ + int iOff; + + assert( p->rc==SQLITE_OK ); + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); + + ASSERT_SZLEAF_OK(pIter->pLeaf); + iOff = pIter->iLeafOffset; + + /* Next entry is on the next page */ + if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + fts5SegIterNextPage(p, pIter); + if( p->rc || pIter->pLeaf==0 ) return; + pIter->iRowid = 0; + iOff = 4; + } + + if( iOffiEndofDoclist ){ + /* Next entry is on the current page */ + i64 iDelta; + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], &iDelta); + pIter->iLeafOffset = iOff; + pIter->iRowid += iDelta; + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ + if( pIter->pSeg ){ + int nKeep = 0; + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); + } + pIter->iLeafOffset = iOff; + fts5SegIterLoadTerm(p, pIter, nKeep); + }else{ + const u8 *pList = 0; + const char *zTerm = 0; + int nList; + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + if( pList==0 ) goto next_none_eof; + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList; + sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + } + + if( pbNewTerm ) *pbNewTerm = 1; + }else{ + goto next_none_eof; + } + + fts5SegIterLoadNPos(p, pIter); + + return; + next_none_eof: + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; +} + + /* ** Advance iterator pIter to the next entry. ** @@ -1752,151 +1873,119 @@ static void fts5SegIterNext( Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ - assert( pbNewTerm==0 || *pbNewTerm==0 ); - if( p->rc==SQLITE_OK ){ - if( pIter->flags & FTS5_SEGITER_REVERSE ){ - assert( pIter->pNextLeaf==0 ); - if( pIter->iRowidOffset>0 ){ - u8 *a = pIter->pLeaf->p; - int iOff; - int nPos; - int bDummy; - i64 iDelta; + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; - pIter->iRowidOffset--; - pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; - fts5SegIterLoadNPos(p, pIter); - iOff = pIter->iLeafOffset; - if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ - iOff += pIter->nPos; - } - fts5GetVarint(&a[iOff], (u64*)&iDelta); - pIter->iRowid -= iDelta; - }else{ - fts5SegIterReverseNewPage(p, pIter); + assert( pbNewTerm==0 || *pbNewTerm==0 ); + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); + + /* Search for the end of the position list within the current page. */ + u8 *a = pLeaf->p; + int n = pLeaf->szLeaf; + + ASSERT_SZLEAF_OK(pLeaf); + iOff = pIter->iLeafOffset + pIter->nPos; + + if( iOffiEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ + bNewTerm = 1; + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); } }else{ - Fts5Data *pLeaf = pIter->pLeaf; - int iOff; - int bNewTerm = 0; - int nKeep = 0; - - /* Search for the end of the position list within the current page. */ - u8 *a = pLeaf->p; - int n = pLeaf->szLeaf; + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); + pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); + } + pIter->iLeafOffset = iOff; + }else if( pIter->pSeg==0 ){ + const u8 *pList = 0; + const char *zTerm = 0; + int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), + (u8*)zTerm); + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; + } + }else{ + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; ASSERT_SZLEAF_OK(pLeaf); - if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ - iOff = pIter->iLeafOffset; - }else{ - iOff = pIter->iLeafOffset + pIter->nPos; - } - - if( iOffiEndofDoclist ); - if( iOff>=pIter->iEndofDoclist ){ - bNewTerm = 1; - if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ - iOff += fts5GetVarint32(&a[iOff], nKeep); - } - }else{ - u64 iDelta; - iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); - pIter->iRowid += iDelta; - assert_nc( iDelta>0 ); - } + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; - }else if( pIter->pSeg==0 ){ - const u8 *pList = 0; - const char *zTerm = 0; - int nList = 0; - assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); - if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ - sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); - } - if( pList==0 ){ - fts5DataRelease(pIter->pLeaf); - pIter->pLeaf = 0; - }else{ - pIter->pLeaf->p = (u8*)pList; - pIter->pLeaf->nn = nList; - pIter->pLeaf->szLeaf = nList; - pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); - pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); - *pbNewTerm = 1; - } - }else{ - iOff = 0; - /* Next entry is not on the current page */ - while( iOff==0 ){ - fts5SegIterNextPage(p, pIter); - pLeaf = pIter->pLeaf; - if( pLeaf==0 ) break; - ASSERT_SZLEAF_OK(pLeaf); - if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ - iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - - if( pLeaf->nn>pLeaf->szLeaf ){ - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( - &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist ); - } + } - } - else if( pLeaf->nn>pLeaf->szLeaf ){ - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( - &pLeaf->p[pLeaf->szLeaf], iOff + } + else if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff ); - pIter->iLeafOffset = iOff; - pIter->iEndofDoclist = iOff; - bNewTerm = 1; - } - assert_nc( iOffszLeaf - || p->pConfig->eDetail==FTS5_DETAIL_NONE - ); - if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - } + pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; + bNewTerm = 1; } + assert_nc( iOffszLeaf ); + if( iOff>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } + } + } - /* Check if the iterator is now at EOF. If so, return early. */ - if( pIter->pLeaf ){ - if( bNewTerm ){ - if( pIter->flags & FTS5_SEGITER_ONETERM ){ - fts5DataRelease(pIter->pLeaf); - pIter->pLeaf = 0; - }else{ - fts5SegIterLoadTerm(p, pIter, nKeep); - fts5SegIterLoadNPos(p, pIter); - if( pbNewTerm ) *pbNewTerm = 1; - } - }else{ - /* The following could be done by calling fts5SegIterLoadNPos(). But - ** this block is particularly performance critical, so equivalent - ** code is inlined. - ** - ** Later: Switched back to fts5SegIterLoadNPos() because it supports - ** detail=none mode. Not ideal. - */ -#if 0 - int nSz; - assert( p->rc==SQLITE_OK ); - fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); - pIter->bDel = (nSz & 0x0001); - pIter->nPos = nSz>>1; - assert_nc( pIter->nPos>=0 ); -#endif - fts5SegIterLoadNPos(p, pIter); - } + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf ){ + if( bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); + fts5SegIterLoadNPos(p, pIter); + if( pbNewTerm ) *pbNewTerm = 1; } + }else{ + /* The following could be done by calling fts5SegIterLoadNPos(). But + ** this block is particularly performance critical, so equivalent + ** code is inlined. + ** + ** Later: Switched back to fts5SegIterLoadNPos() because it supports + ** detail=none mode. Not ideal. + */ + int nSz; + assert( p->rc==SQLITE_OK ); + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + assert_nc( pIter->nPos>=0 ); } } } @@ -2231,6 +2320,8 @@ static void fts5SegIterSeekInit( } } + fts5SegIterSetNext(p, pIter); + /* Either: ** ** 1) an error has occurred, or @@ -2288,7 +2379,7 @@ static void fts5SegIterHashInit( pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); - pIter->iEndofDoclist = pLeaf->nn+1; + pIter->iEndofDoclist = pLeaf->nn; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; @@ -2297,6 +2388,8 @@ static void fts5SegIterHashInit( fts5SegIterLoadNPos(p, pIter); } } + + fts5SegIterSetNext(p, pIter); } /* @@ -2540,7 +2633,7 @@ static void fts5SegIterNextFrom( } do{ - if( bMove ) fts5SegIterNext(p, pIter, 0); + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; @@ -2574,7 +2667,9 @@ static void fts5MultiIterAdvanced( for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ - fts5SegIterNext(p, &pIter->aSeg[iEq], 0); + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, 0); i = pIter->nSeg + iEq; } } @@ -2661,7 +2756,7 @@ static void fts5MultiIterNext( if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ - fts5SegIterNext(p, pSeg, &bNewTerm); + pSeg->xNext(p, pSeg, &bNewTerm); } if( pSeg->pLeaf==0 || bNewTerm @@ -2689,7 +2784,8 @@ static void fts5MultiIterNext2( Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; - fts5SegIterNext(p, pSeg, &bNewTerm); + assert( p->rc==SQLITE_OK ); + pSeg->xNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ @@ -2809,7 +2905,8 @@ static void fts5MultiIterNew( for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - fts5SegIterNext(p, &pNew->aSeg[iEq], 0); + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); fts5MultiIterAdvanced(p, pNew, iEq, iIter); } } @@ -2859,6 +2956,7 @@ static void fts5MultiIterNew2( }else{ pNew->bEof = 1; } + fts5SegIterSetNext(p, pIter); *ppOut = pNew; } diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index b875354534..c6b09d8121 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -24,8 +24,6 @@ ifcapable !fts5 { foreach_detail_mode $testprefix { -if {[detail_is_none]==0} continue - do_execsql_test 1.0 { CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); diff --git a/manifest b/manifest index 5a749962b4..bd4318635f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\swith\sthis\sbranch. -D 2016-01-11T18:30:34.152 +C Improve\sperformance\sof\sfts5\slow\slevel\siterators. +D 2016-01-12T19:28:51.507 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1 +F ext/fts5/fts5_index.c 5ab044a67919e2c9f42f0288a39778136154511e F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb @@ -118,7 +118,7 @@ F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced -F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc +F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2c0b93553ebb00a70c611eb8dac021877933f088 ecc98bef43c2fd07d64e4efddf340929a875ebac -R fac95fe8d4249da246cad11b14ac9297 +P a73d245f2e523a57163bc93d9fb6a74c4cdf21e7 +R 1ce1920656146c835e2df297e09b2c78 U dan -Z 0acdab0d7c32bec7117a883df7ebebf7 +Z 25b8dbc71e1c6971b424b6f8a3c7183e diff --git a/manifest.uuid b/manifest.uuid index 0ab6f5ff10..0aaeafe188 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a73d245f2e523a57163bc93d9fb6a74c4cdf21e7 \ No newline at end of file +bc5118f40a11f64ffb4e1c086277fa80b9764745 \ No newline at end of file From b19e7352c1113b6ae908d7a10d9777a6708de294 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jan 2016 19:37:20 +0000 Subject: [PATCH 33/44] Add the ".vfslist" command to the command-line shell. FossilOrigin-Name: 5727562b75edf25102cd72607c420d245379c96d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b4c264a04b..bd838dc3eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\suse\sof\san\suninitialized\svariable\sfollowing\ssystem\serrors\nin\sthe\smulti-threaded\ssorter. -D 2016-01-12T14:10:05.866 +C Add\sthe\s".vfslist"\scommand\sto\sthe\scommand-line\sshell. +D 2016-01-12T19:37:20.761 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -334,7 +334,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 372234e3554a7917544242563216af760d7d4219 -F src/shell.c ed71dc7679e6f087a3f1ea3f9dae4b0fae7209c3 +F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4 F src/sqlite.h.in 7d87d71b9a4689c51fa092f48f16590ff71558e3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5fc0a4c1f353c4c06fcbc4bce8bbe1897ef49eee -R 92b34da2a8b41bb1db044333a3c02410 +P 4bb2c1df105c2d21f5c4c7ef656ff1d6e15f78bc +R 74300fef6ffb708c0011be0eb0839258 U drh -Z e6a278c031968913cb3469aed14f2f8a +Z 0c47ae55d5f3f2500ab8561940e3da5a diff --git a/manifest.uuid b/manifest.uuid index 9e4a59f983..9a5df7152e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4bb2c1df105c2d21f5c4c7ef656ff1d6e15f78bc \ No newline at end of file +5727562b75edf25102cd72607c420d245379c96d \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index b7a7abcd33..3f8b22f4fa 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1921,6 +1921,7 @@ static char zHelp[] = ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" ".vfsinfo ?AUX? Information about the top-level VFS\n" + ".vfslist List all available VFSes\n" ".vfsname ?AUX? Print the name of the VFS stack\n" ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" " Negative values right-justify\n" @@ -4170,6 +4171,24 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + sqlite3_vfs *pVfs; + sqlite3_vfs *pCurrent = 0; + if( p->db ){ + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); + } + for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ + utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + if( pVfs->pNext ){ + raw_printf(p->out, "-----------------------------------\n"); + } + } + }else + if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; From a04a8be2c29221f7b69de3a22fb8fce27e86a8e5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Jan 2016 17:50:10 +0000 Subject: [PATCH 34/44] When the [https://www.sqlite.org/queryplanner.html#partialsort|block sorting optimization] is used in a scalar subquery, be sure to exit the loop as soon as the first valid output row is received. Fix for ticket [cb3aa0641d9a4]. FossilOrigin-Name: cdbb0947f9ce18d6d7e29ffab5ea6a2ee5365fbb --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 21 ++++++++++++--------- test/orderby1.test | 17 +++++++++++++++++ 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 33ebb02266..141b5d8f8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"detail"\soption\sto\sfts5.\sUsed\sto\sreduce\sthe\samount\sof\sinformation\sstored\sin\san\sfts5\sindex. -D 2016-01-12T19:45:01.185 +C When\sthe\s[https://www.sqlite.org/queryplanner.html#partialsort|block\ssorting\soptimization]\nis\sused\sin\sa\sscalar\ssubquery,\sbe\ssure\sto\sexit\sthe\sloop\sas\ssoon\sas\sthe\sfirst\nvalid\soutput\srow\sis\sreceived.\s\sFix\sfor\sticket\s[cb3aa0641d9a4]. +D 2016-01-13T17:50:10.253 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -338,7 +338,7 @@ F src/printf.c af589a27b7d40f6f4f704e9eea99f02f18ad6d32 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 372234e3554a7917544242563216af760d7d4219 +F src/select.c 5b0f2aa9f73ec7b65d1711d485471854d5bad23c F src/shell.c dcd7a83645ef2a58ee9c6d0ea4714d877d7835c4 F src/sqlite.h.in 7d87d71b9a4689c51fa092f48f16590ff71558e3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -908,7 +908,7 @@ F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 F test/offset1.test f06b83657bcf26f9ce805e67450e189e282143b2 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 870e150450437d3980badbde3d0166b81d9e33f6 +F test/orderby1.test 4d22a7c75f6a83fc1f188cc7bb5192285fdf2552 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1412,8 +1412,8 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5727562b75edf25102cd72607c420d245379c96d bc5118f40a11f64ffb4e1c086277fa80b9764745 -R 32b6994fdd76448ebd10466a9fb05893 -T +closed bc5118f40a11f64ffb4e1c086277fa80b9764745 -U dan -Z d6aecf5b74614ca62cb2aeb2f23739db +P a220e85fe535af5ef2da6ef5fb76abe5a96b5abf +Q -2a74129a21f9745f1363f844807e2d10201a3f40 +R 413681dc8f5cf3f3175ce236e8b6c97f +U drh +Z fd0638b6043efd9e3f580e2c8b2c7054 diff --git a/manifest.uuid b/manifest.uuid index 26a759aaef..1133ef1cfb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a220e85fe535af5ef2da6ef5fb76abe5a96b5abf \ No newline at end of file +cdbb0947f9ce18d6d7e29ffab5ea6a2ee5365fbb \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0e7c4595d1..fd094d05dd 100644 --- a/src/select.c +++ b/src/select.c @@ -54,6 +54,7 @@ struct SortCtx { int regReturn; /* Register holding block-output return address */ int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -516,6 +517,7 @@ static void pushOntoSorter( int regRecord = ++pParse->nMem; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); assert( nData==1 || regData==regOrigData ); @@ -526,6 +528,9 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ @@ -534,7 +539,6 @@ static void pushOntoSorter( if( nPrefixReg==0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -569,6 +573,10 @@ static void pushOntoSorter( pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addrFirst); sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); @@ -579,14 +587,8 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( pSelect->iLimit ){ + if( iLimit ){ int addr; - int iLimit; - if( pSelect->iOffset ){ - iLimit = pSelect->iOffset+1; - }else{ - iLimit = pSelect->iLimit; - } addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); @@ -1190,7 +1192,7 @@ static void generateSortTail( SelectDest *pDest /* Write the sorted results here */ ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int addrOnce = 0; @@ -1209,6 +1211,7 @@ static void generateSortTail( struct ExprList_item *aOutEx = p->pEList->a; #endif + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeGoto(v, addrBreak); diff --git a/test/orderby1.test b/test/orderby1.test index 3e785c54e4..831936ae96 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -528,4 +528,21 @@ do_test 8.3 { set res } 5000 +#--------------------------------------------------------------------------- +# https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029 +# +# Adverse interaction between scalar subqueries and the partial-sorting +# logic. +# +do_execsql_test 9.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(y); + INSERT INTO t2 VALUES(9),(8),(3),(4); + SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y); +} {13} + + finish_test From 504ef440224126ca19756bdba9b8eeb877357674 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Jan 2016 18:06:08 +0000 Subject: [PATCH 35/44] Add documentation for the SQLITE_FCNTL_JOURNAL_POINTER file-control. FossilOrigin-Name: 807c5aee2e5e5c01eb33688a199755c345c3664e --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 9 +++++++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 183b7bbcf7..8ad71efa12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\sSQLITE_FCNTL_JOURNAL_POINTER\sfile\scontrol\sfor\sobtaining\nthe\ssqlite3_file\spointer\sassociated\swith\sa\srollback\sjournal\sor\sWAL\sfile. -D 2016-01-08T02:27:01.544 +C Add\sdocumentation\sfor\sthe\sSQLITE_FCNTL_JOURNAL_POINTER\sfile-control. +D 2016-01-13T18:06:08.461 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -335,7 +335,7 @@ F src/resolve.c a83b41104e6ff69855d03cd0aaa09e93927ec39f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f8fded11fc443a9f5a73cc5db069d06b34460e2f F src/shell.c ed71dc7679e6f087a3f1ea3f9dae4b0fae7209c3 -F src/sqlite.h.in f18858fc9440fd3c399b733057d12c53a2360d11 +F src/sqlite.h.in 214476a62012e578f42133a9a3b4f97a9aa421a3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d F src/sqliteInt.h 987b46b1d8089155caecbdb14b78850ae1e198ff @@ -1406,10 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P af5c7714e993f060841f1e893f754ddf3870e6d0 -R 1ed70ce93210d32572be9b1ea5c549df -T *branch * file-control-journal -T *sym-file-control-journal * -T -sym-trunk * +P a02ace9a746ff02fc1a5969bf13939b568624f93 +R 0f2c35be54361ea831426d444981cf27 U drh -Z 34b65576034eb21a6f6ec3e5456f3b6a +Z f9a8aaf9aacff558b761ee4baaf7cc0d diff --git a/manifest.uuid b/manifest.uuid index 8d6b249f6a..a1eb8cfe4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a02ace9a746ff02fc1a5969bf13939b568624f93 \ No newline at end of file +807c5aee2e5e5c01eb33688a199755c345c3664e \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2f61704a2d..59b30cdd3a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -794,8 +794,13 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database -** connection. See the [sqlite3_file_control()] documentation for -** additional information. +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +**
  • [[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** **
  • [[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. From 7f50d693598ae3c58f57200c75f63eda492c513d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jan 2016 18:25:36 +0000 Subject: [PATCH 36/44] Add documentation for new API functions to the comments in fts5.h. FossilOrigin-Name: 14d296fa7b43b4999a1fd070cf030f83fe1b94c3 --- ext/fts5/fts5.h | 51 ++++++++++++++++++++++++++++++++++++++++++++++++- manifest | 15 +++++++-------- manifest.uuid | 2 +- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 990c3f2f0d..96ecb38e33 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -84,6 +84,9 @@ struct Fts5PhraseIter { ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer @@ -104,6 +107,11 @@ struct Fts5PhraseIter { ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument @@ -119,6 +127,9 @@ struct Fts5PhraseIter { ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** ** xRowid: ** Returns the rowid of the current row. ** @@ -210,10 +221,48 @@ struct Fts5PhraseIter { ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above -** with the xPhraseFirst() and xPhraseNext() API methods. +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** ** xPhraseNext() ** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ diff --git a/manifest b/manifest index 19016057ef..a97cf05cad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_FCNTL_JOURNAL_POINTER\sfile\scontrol. -D 2016-01-13T18:12:44.833 +C Add\sdocumentation\sfor\snew\sAPI\sfunctions\sto\sthe\scomments\sin\sfts5.h. +D 2016-01-13T18:25:36.824 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -96,7 +96,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 +F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 @@ -1412,8 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cdbb0947f9ce18d6d7e29ffab5ea6a2ee5365fbb 807c5aee2e5e5c01eb33688a199755c345c3664e -R 9f69ee1beb680eb7753159da9e58457d -T +closed 807c5aee2e5e5c01eb33688a199755c345c3664e -U drh -Z 158e43f82215274b7eb57ca9b5089c6b +P ad3daa5427672d2d236939ad3285b765f4601801 +R f25d53bc8dd0e47144b4d953a312dda2 +U dan +Z 9aba901b3a90a935129ccd9da6e0e571 diff --git a/manifest.uuid b/manifest.uuid index 92b929ad75..73dacf638f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad3daa5427672d2d236939ad3285b765f4601801 \ No newline at end of file +14d296fa7b43b4999a1fd070cf030f83fe1b94c3 \ No newline at end of file From 13c808a9fca0775fec27a64701b940d5f2c16934 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Jan 2016 21:23:48 +0000 Subject: [PATCH 37/44] Look for pthread_mutexattr_init() instead of pthread_create() in autoconf scripts, to work around issues with ASAN. FossilOrigin-Name: 9c3a0dc2e8bf202cd846841bd5d4ea67aa189c1a --- autoconf/configure.ac | 2 +- configure | 24 ++++++++++++------------ configure.ac | 2 +- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 8df915e955..deba91ae1d 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -73,7 +73,7 @@ AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" - AC_SEARCH_LIBS(pthread_create, pthread) + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi AC_SUBST(THREADSAFE_FLAGS) #----------------------------------------------------------------------- diff --git a/configure b/configure index 850d986306..951b719ed5 100755 --- a/configure +++ b/configure @@ -10464,9 +10464,9 @@ fi if test "$SQLITE_THREADSAFE" = "1"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 -$as_echo_n "checking for library containing pthread_create... " >&6; } -if ${ac_cv_search_pthread_create+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 +$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS @@ -10479,11 +10479,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #ifdef __cplusplus extern "C" #endif -char pthread_create (); +char pthread_mutexattr_init (); int main () { -return pthread_create (); +return pthread_mutexattr_init (); ; return 0; } @@ -10496,25 +10496,25 @@ for ac_lib in '' pthread; do LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_pthread_create=$ac_res + ac_cv_search_pthread_mutexattr_init=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext - if ${ac_cv_search_pthread_create+:} false; then : + if ${ac_cv_search_pthread_mutexattr_init+:} false; then : break fi done -if ${ac_cv_search_pthread_create+:} false; then : +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : else - ac_cv_search_pthread_create=no + ac_cv_search_pthread_mutexattr_init=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 -$as_echo "$ac_cv_search_pthread_create" >&6; } -ac_res=$ac_cv_search_pthread_create +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 +$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } +ac_res=$ac_cv_search_pthread_mutexattr_init if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" diff --git a/configure.ac b/configure.ac index 0b94d33930..26a60147dd 100644 --- a/configure.ac +++ b/configure.ac @@ -194,7 +194,7 @@ fi AC_SUBST(SQLITE_THREADSAFE) if test "$SQLITE_THREADSAFE" = "1"; then - AC_SEARCH_LIBS(pthread_create, pthread) + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi ########## diff --git a/manifest b/manifest index a97cf05cad..db7623fdf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdocumentation\sfor\snew\sAPI\sfunctions\sto\sthe\scomments\sin\sfts5.h. -D 2016-01-13T18:25:36.824 +C Look\sfor\spthread_mutexattr_init()\sinstead\sof\spthread_create()\sin\sautoconf\nscripts,\sto\swork\saround\sissues\swith\sASAN. +D 2016-01-13T21:23:48.485 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 089e5ecdb5761e64ea1013ded02feb4d8b29927d F autoconf/README 14458f1046c118efa721aadec5f227e876d3cd38 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/configure.ac 82c628c68f0f204d57343dc345b6dbd1789fb1bd +F autoconf/configure.ac 2f41055b10873cfe443a08185343e38c47e2f9f0 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -29,8 +29,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 42b71ad3fe21c9e88fa59e8458ca1a6bc72eb0c0 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure b4519bb54fff37e7cde9d03d8f2946f18d5d0086 x -F configure.ac fcfc67b323d32daaa3e46cf7782d9465ed423a6d +F configure d57d3b9d5c66549e54a4a2960de9813142d30a5a x +F configure.ac c59513d560b3107995c73b9cc1e55bfd086c4764 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad3daa5427672d2d236939ad3285b765f4601801 -R f25d53bc8dd0e47144b4d953a312dda2 -U dan -Z 9aba901b3a90a935129ccd9da6e0e571 +P 14d296fa7b43b4999a1fd070cf030f83fe1b94c3 +R 2b677537f1e9b18e2947e5670c77e061 +U drh +Z d3d675fa0c765d73d93e44db2a68c69e diff --git a/manifest.uuid b/manifest.uuid index 73dacf638f..9104c2d889 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14d296fa7b43b4999a1fd070cf030f83fe1b94c3 \ No newline at end of file +9c3a0dc2e8bf202cd846841bd5d4ea67aa189c1a \ No newline at end of file From b4acd6a8d64dfb0f768b39c08fb37b3c03b27156 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Jan 2016 01:56:43 +0000 Subject: [PATCH 38/44] Fix the build so that it works for SQLITE_OMIT_WAL again. FossilOrigin-Name: d55a557bc70460c46a67ae5acd9f8a8356a580c2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index db7623fdf9..d9cc140750 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Look\sfor\spthread_mutexattr_init()\sinstead\sof\spthread_create()\sin\sautoconf\nscripts,\sto\swork\saround\sissues\swith\sASAN. -D 2016-01-13T21:23:48.485 +C Fix\sthe\sbuild\sso\sthat\sit\sworks\sfor\sSQLITE_OMIT_WAL\sagain. +D 2016-01-14T01:56:43.015 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -325,7 +325,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c b509b49b40a269e7b75ab511b6e92b2dc9444359 F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 5dda328a9a5e9405b6971d62d0d5e3ea85c6195a +F src/pager.c f4e9ac39fbb1e0fde97af85c0f4e00eb90764b67 F src/pager.h 1c2a49143dfba9e69cc8159ef019f472ed8d260b F src/parse.y caad1e98edeca6960493d0c60d31b76820dd7776 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 14d296fa7b43b4999a1fd070cf030f83fe1b94c3 -R 2b677537f1e9b18e2947e5670c77e061 +P 9c3a0dc2e8bf202cd846841bd5d4ea67aa189c1a +R 7d3ddebf3af55be615ba3903145bc66f U drh -Z d3d675fa0c765d73d93e44db2a68c69e +Z 40f341244fdfbe64f691992440e887ec diff --git a/manifest.uuid b/manifest.uuid index 9104c2d889..6e8776e5e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c3a0dc2e8bf202cd846841bd5d4ea67aa189c1a \ No newline at end of file +d55a557bc70460c46a67ae5acd9f8a8356a580c2 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8f49e656e8..2c904d2df1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5627,7 +5627,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( rc!=SQLITE_OK ){ return rc; } - sqlite3WalExclusiveMode(pPager->pWal, 1); + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); } /* Grab the write lock on the log file. If successful, upgrade to @@ -6697,7 +6697,11 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif } /* From 62e63bb9a93732b7a47eba95d90707c3d10cac31 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Jan 2016 12:23:16 +0000 Subject: [PATCH 39/44] Fix harmless compiler warnings in sqldiff.c. FossilOrigin-Name: 74808a79ea3194f6cc00084b9dccc380846a6051 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqldiff.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d9cc140750..7187aa4558 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sso\sthat\sit\sworks\sfor\sSQLITE_OMIT_WAL\sagain. -D 2016-01-14T01:56:43.015 +C Fix\sharmless\scompiler\swarnings\sin\ssqldiff.c. +D 2016-01-14T12:23:16.611 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -1401,7 +1401,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 653db1a9294e0b34849c4504599c282a18d8b8c6 +F tool/sqldiff.c 5a26205111e6fa856d9b1535b1637744dcdb930b F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9c3a0dc2e8bf202cd846841bd5d4ea67aa189c1a -R 7d3ddebf3af55be615ba3903145bc66f +P d55a557bc70460c46a67ae5acd9f8a8356a580c2 +R 34645f4f80a93ccb4b9986535bb6f0b0 U drh -Z 40f341244fdfbe64f691992440e887ec +Z 0c8aaf707fea483adf01b6983fc067a8 diff --git a/manifest.uuid b/manifest.uuid index 6e8776e5e5..4d3f464471 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d55a557bc70460c46a67ae5acd9f8a8356a580c2 \ No newline at end of file +74808a79ea3194f6cc00084b9dccc380846a6051 \ No newline at end of file diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 56ff53ee25..ae01cd3c4d 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -994,7 +994,7 @@ static int rbuDeltaCreate( zDelta += lenOut; putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; - return zDelta - zOrigDelta; + return (int)(zDelta - zOrigDelta); } /* Compute the hash table used to locate matching sections in the @@ -1141,7 +1141,7 @@ static int rbuDeltaCreate( putInt(checksum(zOut, lenOut), &zDelta); *(zDelta++) = ';'; sqlite3_free(collide); - return zDelta - zOrigDelta; + return (int)(zDelta - zOrigDelta); } /* From 7647377f528ab8fa26ba4be05a5151a51ffbbf08 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Jan 2016 13:22:24 +0000 Subject: [PATCH 40/44] Fix harmless compiler warnings in RBU and add RBU to the autoconf and windows makefiles. FossilOrigin-Name: fabe78c5d8ab353988f6fe0decacd651edc17ec2 --- Makefile.in | 11 ++++++++++- Makefile.msc | 9 ++++++++- ext/rbu/rbu.c | 7 +++---- ext/rbu/sqlite3rbu.c | 37 +++++++++++++++++++------------------ manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 50 insertions(+), 34 deletions(-) diff --git a/Makefile.in b/Makefile.in index 1fe49d6789..2782e765f5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -179,7 +179,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1028,6 +1028,9 @@ fts5.c: $(FTS5_SRC) fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c + # Rules to build the 'testfixture' application. # @@ -1131,6 +1134,12 @@ wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo + $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) + +loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la + $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) + # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. diff --git a/Makefile.msc b/Makefile.msc index 92561a8a32..0eb660adc7 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -839,7 +839,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1714,6 +1714,9 @@ fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c + # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test @@ -1836,6 +1839,10 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) + $(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ + $(LDFLAGS) $(LTLINKOPTS) + clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index dd46743def..018296b8dd 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -75,7 +75,7 @@ int main(int argc, char **argv){ /* Process command line arguments. Following this block local variables ** zTarget, zRbu and nStep are all set. */ if( argc==5 ){ - int nArg1 = strlen(argv[1]); + size_t nArg1 = strlen(argv[1]); if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); nStep = atoi(argv[2]); }else if( argc!=3 ){ @@ -103,7 +103,7 @@ int main(int argc, char **argv){ "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", nProgress ); - fprintf(stdout, zBuf); + fprintf(stdout, "%s", zBuf); break; case SQLITE_DONE: @@ -111,7 +111,7 @@ int main(int argc, char **argv){ "SQLITE_DONE: rbu update completed (%lld operations)\n", nProgress ); - fprintf(stdout, zBuf); + fprintf(stdout, "%s", zBuf); break; default: @@ -122,4 +122,3 @@ int main(int argc, char **argv){ sqlite3_free(zErrmsg); return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1; } - diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 4c38e14c9e..145b446d30 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -935,7 +935,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ p->rc = SQLITE_NOMEM; }else{ @@ -981,8 +981,8 @@ static char *rbuStrndup(const char *zStr, int *pRc){ assert( *pRc==SQLITE_OK ); if( zStr ){ - int nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc(nCopy); + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); if( zRet ){ memcpy(zRet, zStr, nCopy); }else{ @@ -2330,7 +2330,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->nFrame==pRbu->nFrameAlloc ){ int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; RbuFrame *aNew; - aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); if( aNew==0 ) return SQLITE_NOMEM; pRbu->aFrame = aNew; pRbu->nFrameAlloc = nNew; @@ -2395,7 +2395,7 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ if( nChar==0 ){ return 0; } - zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); + zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } @@ -3029,11 +3029,12 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ){ sqlite3rbu *p; - int nTarget = strlen(zTarget); - int nRbu = strlen(zRbu); - int nState = zState ? strlen(zState) : 0; + size_t nTarget = strlen(zTarget); + size_t nRbu = strlen(zRbu); + size_t nState = zState ? strlen(zState) : 0; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; - p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1); + p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ RbuState *pState = 0; @@ -3170,7 +3171,7 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ int i; - int nErrmsg = strlen(p->zErrmsg); + size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ int nDel = 8; @@ -3634,7 +3635,7 @@ static int rbuVfsShmMap( if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ if( iRegion<=p->nShm ){ int nByte = (iRegion+1) * sizeof(char*); - char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); if( apNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -3645,7 +3646,7 @@ static int rbuVfsShmMap( } if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ - char *pNew = (char*)sqlite3_malloc(szRegion); + char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -3755,7 +3756,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = strlen(zName); + int n = (int)strlen(zName); const char *z = &zName[n]; if( flags & SQLITE_OPEN_URI ){ int odd = 0; @@ -3781,8 +3782,8 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - int nCopy = strlen(zName); - char *zCopy = sqlite3_malloc(nCopy+2); + size_t nCopy = strlen(zName); + char *zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ memcpy(zCopy, zName, nCopy); zCopy[nCopy-3] = 'o'; @@ -4011,13 +4012,13 @@ int sqlite3rbu_create_vfs(const char *zName, const char *zParent){ }; rbu_vfs *pNew = 0; /* Newly allocated VFS */ - int nName; int rc = SQLITE_OK; + size_t nName; + size_t nByte; - int nByte; nName = strlen(zName); nByte = sizeof(rbu_vfs) + nName + 1; - pNew = (rbu_vfs*)sqlite3_malloc(nByte); + pNew = (rbu_vfs*)sqlite3_malloc64(nByte); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/manifest b/manifest index 7187aa4558..566a67eeac 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sharmless\scompiler\swarnings\sin\ssqldiff.c. -D 2016-01-14T12:23:16.611 -F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 +C Fix\sharmless\scompiler\swarnings\sin\sRBU\sand\sadd\sRBU\sto\sthe\sautoconf\sand\swindows\nmakefiles. +D 2016-01-14T13:22:24.150 +F Makefile.in cfa1ac03c4b414992fd53f24d978b45b0c21de55 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 +F Makefile.msc 1dc36cfd3c047c9685e539257e158485f16ef035 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -208,7 +208,7 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/rbu/rbu.c e572f7ddef2ef3a73d03e7b44d36448e466772b7 +F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 F ext/rbu/rbu1.test 57601977588603e82700a43c279bd55282ffa482 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 @@ -230,7 +230,7 @@ F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c ea47de615e911b3a69a8e7fb3be3866298403a25 +F ext/rbu/sqlite3rbu.c bea954197524631f2691ec272e8a42df8cad01cc F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d55a557bc70460c46a67ae5acd9f8a8356a580c2 -R 34645f4f80a93ccb4b9986535bb6f0b0 +P 74808a79ea3194f6cc00084b9dccc380846a6051 +R e29cff8691de1217f1757056bcfdb892 U drh -Z 0c8aaf707fea483adf01b6983fc067a8 +Z 1e6a858f101158e4efde76213bfd70e5 diff --git a/manifest.uuid b/manifest.uuid index 4d3f464471..6ea6f0c03c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74808a79ea3194f6cc00084b9dccc380846a6051 \ No newline at end of file +fabe78c5d8ab353988f6fe0decacd651edc17ec2 \ No newline at end of file From f705e9deab4a18e9c5fd726a59226329e4674c95 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Jan 2016 14:15:54 +0000 Subject: [PATCH 41/44] Fix compiler warnings in fts5. FossilOrigin-Name: 5a343cc0336bba056df4449e6cd2e3fb9e75a105 --- ext/fts5/fts5Int.h | 1 - ext/fts5/fts5_config.c | 2 +- ext/fts5/fts5_expr.c | 5 +++-- ext/fts5/fts5_index.c | 4 +--- ext/fts5/fts5_main.c | 7 ------- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- 7 files changed, 17 insertions(+), 26 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1a3d57ca26..2a16ec065f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -424,7 +424,6 @@ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ -u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int); int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); /* diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index fd6da7f11e..d9778bca20 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -379,7 +379,7 @@ static int fts5ConfigParseSpecial( { 0, 0 } }; - if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){ + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ *pzErr = sqlite3_mprintf("malformed detail=... directive"); } return rc; diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 16209bf4ba..409fbd1d05 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2428,7 +2428,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ return 0; } } - return 1; + break; } case FTS5_OR: { @@ -2453,10 +2453,11 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ fts5ExprClearPoslists(pNode); return 0; } - return 1; + break; } } } + return 1; } void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 2488cd7ca9..cc5b8ea521 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1772,8 +1772,6 @@ static void fts5SegIterNext_Reverse( if( pIter->iRowidOffset>0 ){ u8 *a = pIter->pLeaf->p; int iOff; - int nPos; - int bDummy; i64 iDelta; pIter->iRowidOffset--; @@ -1821,7 +1819,7 @@ static void fts5SegIterNext_None( if( iOffiEndofDoclist ){ /* Next entry is on the current page */ i64 iDelta; - iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], &iDelta); + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index ba8903c9a7..fec589efcb 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -310,13 +310,6 @@ static int fts5IsContentless(Fts5Table *pTab){ return pTab->pConfig->eContent==FTS5_CONTENT_NONE; } -/* -** Return true if pTab is an offsetless table. -*/ -static int fts5IsOffsetless(Fts5Table *pTab){ - return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL; -} - /* ** Delete a virtual table handle allocated by fts5InitVtab(). */ diff --git a/manifest b/manifest index 566a67eeac..7290cda6dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sRBU\sand\sadd\sRBU\sto\sthe\sautoconf\sand\swindows\nmakefiles. -D 2016-01-14T13:22:24.150 +C Fix\scompiler\swarnings\sin\sfts5. +D 2016-01-14T14:15:54.059 F Makefile.in cfa1ac03c4b414992fd53f24d978b45b0c21de55 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1dc36cfd3c047c9685e539257e158485f16ef035 @@ -97,14 +97,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 +F ext/fts5/fts5Int.h 3a09b2eaeac2860d7afc59843796a9410f818ebf F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 -F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285 +F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217 +F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 5ab044a67919e2c9f42f0288a39778136154511e -F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e +F ext/fts5/fts5_index.c 11e2b566b8b29fb42bb970969c92f3a33c2c1125 +F ext/fts5/fts5_main.c 488ceecdb4400ecc6a3d3b2247cedef153955388 F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 74808a79ea3194f6cc00084b9dccc380846a6051 -R e29cff8691de1217f1757056bcfdb892 -U drh -Z 1e6a858f101158e4efde76213bfd70e5 +P fabe78c5d8ab353988f6fe0decacd651edc17ec2 +R fee981891bfd69d76d7d5d050236b09c +U dan +Z 4d3da4cf65c48965923bd47b765f8e62 diff --git a/manifest.uuid b/manifest.uuid index 6ea6f0c03c..3b2733f52b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fabe78c5d8ab353988f6fe0decacd651edc17ec2 \ No newline at end of file +5a343cc0336bba056df4449e6cd2e3fb9e75a105 \ No newline at end of file From 17535393b9363702fd8fda7a5a7b75446a2a146a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Jan 2016 14:19:50 +0000 Subject: [PATCH 42/44] Fix another compiler warning in fts5 code. FossilOrigin-Name: d17bc2c92f4d086280e49a3cc72993be7fee2da7 --- ext/fts5/fts5_index.c | 9 +-------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index cc5b8ea521..4368128796 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2001,7 +2001,6 @@ static void fts5SegIterNext( ** the doclist. */ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ - int eDetail = p->pConfig->eDetail; Fts5DlidxIter *pDlidx = pIter->pDlidx; Fts5Data *pLast = 0; int pgnoLast = 0; @@ -2016,11 +2015,6 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ -#if 0 - if( eDetail!=FTS5_DETAIL_NONE ){ - pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); - } -#else int iPoslist; if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ iPoslist = pIter->iTermLeafOffset; @@ -2028,11 +2022,10 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ iPoslist = 4; } fts5IndexSkipVarint(pLeaf->p, iPoslist); - assert( eDetail==FTS5_DETAIL_NONE || iPoslist==( + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==( pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) )); pIter->iLeafOffset = iPoslist; -#endif /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to diff --git a/manifest b/manifest index 7290cda6dc..2748e213ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\sfts5. -D 2016-01-14T14:15:54.059 +C Fix\sanother\scompiler\swarning\sin\sfts5\scode. +D 2016-01-14T14:19:50.419 F Makefile.in cfa1ac03c4b414992fd53f24d978b45b0c21de55 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1dc36cfd3c047c9685e539257e158485f16ef035 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217 F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 11e2b566b8b29fb42bb970969c92f3a33c2c1125 +F ext/fts5/fts5_index.c 140617e34ccc61c2af5d313ecea6bca17591d185 F ext/fts5/fts5_main.c 488ceecdb4400ecc6a3d3b2247cedef153955388 F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fabe78c5d8ab353988f6fe0decacd651edc17ec2 -R fee981891bfd69d76d7d5d050236b09c +P 5a343cc0336bba056df4449e6cd2e3fb9e75a105 +R 15e5cdb9315970f41cbec5989792ed12 U dan -Z 4d3da4cf65c48965923bd47b765f8e62 +Z 4127a035afc4dee8202030a933b1de27 diff --git a/manifest.uuid b/manifest.uuid index 3b2733f52b..ef7de38a98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a343cc0336bba056df4449e6cd2e3fb9e75a105 \ No newline at end of file +d17bc2c92f4d086280e49a3cc72993be7fee2da7 \ No newline at end of file From d625adb8dbf0558b8575a9e6ee5d0e74489253ef Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Jan 2016 14:29:57 +0000 Subject: [PATCH 43/44] Fix more compiler warnings in FTS5. FossilOrigin-Name: f791fc7009d68941c60d7bf953bab19d86112133 --- ext/fts5/fts5Int.h | 23 +++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 2a16ec065f..1daeefc09f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -339,6 +339,29 @@ int sqlite3Fts5IndexClose(Fts5Index *p); ** } */ +/* +** Return a simple checksum value based on the arguments. +*/ +u64 sqlite3Fts5IndexEntryCksum( + i64 iRowid, + int iCol, + int iPos, + int iIdx, + const char *pTerm, + int nTerm +); + +/* +** Argument p points to a buffer containing utf-8 text that is n bytes in +** size. Return the number of bytes in the nChar character prefix of the +** buffer, or 0 if there are less than nChar characters in total. +*/ +int sqlite3Fts5IndexCharlenToBytelen( + const char *p, + int nByte, + int nChar +); + /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. diff --git a/manifest b/manifest index 2748e213ba..7832158c65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\scompiler\swarning\sin\sfts5\scode. -D 2016-01-14T14:19:50.419 +C Fix\smore\scompiler\swarnings\sin\sFTS5. +D 2016-01-14T14:29:57.029 F Makefile.in cfa1ac03c4b414992fd53f24d978b45b0c21de55 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1dc36cfd3c047c9685e539257e158485f16ef035 @@ -97,7 +97,7 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 3a09b2eaeac2860d7afc59843796a9410f818ebf +F ext/fts5/fts5Int.h 6c5a332e6add01dd69166a252d1818fb75c42a08 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217 @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5a343cc0336bba056df4449e6cd2e3fb9e75a105 -R 15e5cdb9315970f41cbec5989792ed12 -U dan -Z 4127a035afc4dee8202030a933b1de27 +P d17bc2c92f4d086280e49a3cc72993be7fee2da7 +R 3a1d2f76daaee8302a3c8f4c76ae20d1 +U drh +Z 3fe0c32146b166753bb44c9b01cd24a5 diff --git a/manifest.uuid b/manifest.uuid index ef7de38a98..b53571837f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d17bc2c92f4d086280e49a3cc72993be7fee2da7 \ No newline at end of file +f791fc7009d68941c60d7bf953bab19d86112133 \ No newline at end of file From 0ea94db612e0f2348b45dafcedf21d9a8f68c60b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Jan 2016 14:33:36 +0000 Subject: [PATCH 44/44] Yet another change to FTS5 trying to get it to merge successfully into sessions. FossilOrigin-Name: 8dedff3b9ac3e6bf9c131fee19f7d26dc1ebd61f --- ext/fts5/fts5_index.c | 3 +-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 4368128796..2f5b3bdaae 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4856,7 +4856,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ -static int sqlite3Fts5IndexCharlenToBytelen( +int sqlite3Fts5IndexCharlenToBytelen( const char *p, int nByte, int nChar @@ -6092,4 +6092,3 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ } return rc; } - diff --git a/manifest b/manifest index 7832158c65..1354db7a40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smore\scompiler\swarnings\sin\sFTS5. -D 2016-01-14T14:29:57.029 +C Yet\sanother\schange\sto\sFTS5\strying\sto\sget\sit\sto\smerge\ssuccessfully\sinto\ssessions. +D 2016-01-14T14:33:36.733 F Makefile.in cfa1ac03c4b414992fd53f24d978b45b0c21de55 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 1dc36cfd3c047c9685e539257e158485f16ef035 @@ -103,7 +103,7 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217 F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 140617e34ccc61c2af5d313ecea6bca17591d185 +F ext/fts5/fts5_index.c aa798d3a839847fd351b3d0f49520f190e57c2e3 F ext/fts5/fts5_main.c 488ceecdb4400ecc6a3d3b2247cedef153955388 F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb @@ -1412,7 +1412,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d17bc2c92f4d086280e49a3cc72993be7fee2da7 -R 3a1d2f76daaee8302a3c8f4c76ae20d1 +P f791fc7009d68941c60d7bf953bab19d86112133 +R 52083946c40dd4f44744cbc0b447540f U drh -Z 3fe0c32146b166753bb44c9b01cd24a5 +Z 9b36451b3b0b6d81cfd6f7c438e3b3aa diff --git a/manifest.uuid b/manifest.uuid index b53571837f..92616336df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f791fc7009d68941c60d7bf953bab19d86112133 \ No newline at end of file +8dedff3b9ac3e6bf9c131fee19f7d26dc1ebd61f \ No newline at end of file