From fe0c3cfee154f8f5f24c10535507f4d8885a3c5c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Jan 2016 19:01:56 +0000 Subject: [PATCH] Changes to run many fts5 tests with detail=none and detail=col tables as well as the default detail=full. Also fixes for the bugs uncovered by running said tests. FossilOrigin-Name: 6322a1d984e7946735bace8a069ef24b31754b3b --- ext/fts5/fts5Int.h | 1 + ext/fts5/fts5_expr.c | 62 +++++++++++++ ext/fts5/fts5_index.c | 128 +++++++++++++++++---------- ext/fts5/fts5_main.c | 76 +++++++++++----- ext/fts5/test/fts5_common.tcl | 27 ++++++ ext/fts5/test/fts5aa.test | 60 +++++++------ ext/fts5/test/fts5ab.test | 28 +++--- ext/fts5/test/fts5ac.test | 154 +++++++++++++-------------------- ext/fts5/test/fts5ad.test | 32 +++---- ext/fts5/test/fts5ae.test | 52 ++++++----- ext/fts5/test/fts5af.test | 56 ++++++------ ext/fts5/test/fts5detail.test | 31 ++++++- ext/fts5/test/fts5simple2.test | 106 +++++++++++++++++++++++ manifest | 36 ++++---- manifest.uuid | 2 +- 15 files changed, 567 insertions(+), 284 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4ef99ebae1..c369687042 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -646,6 +646,7 @@ Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int ); +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ebd767a617..0a74c7884a 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2327,6 +2327,68 @@ int sqlite3Fts5ExprPopulatePoslists( ); } +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ + pNode->pNear->apPhrase[0]->poslist.n = 0; + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprClearPoslists(pNode->apChild[i]); + } + } +} + +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ + if( pNode ){ + pNode->iRowid = iRowid; + pNode->bEof = 0; + switch( pNode->eType ){ + case FTS5_TERM: + case FTS5_STRING: + return (pNode->pNear->apPhrase[0]->poslist.n>0); + + case FTS5_AND: { + int i; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ + fts5ExprClearPoslists(pNode); + return 0; + } + } + return 1; + } + + case FTS5_OR: { + int i; + int bRet = 0; + for(i=0; inChild; i++){ + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ + bRet = 1; + } + } + if( bRet==0 ){ + fts5ExprClearPoslists(pNode); + } + return bRet; + } + + default: { + assert( pNode->eType==FTS5_NOT ); + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) + ){ + fts5ExprClearPoslists(pNode); + return 0; + } + return 1; + } + } + } +} +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); +} + /* ** This function is only called for detail=columns tables. */ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ef27d73c31..49d8638f33 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1494,12 +1494,13 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ int iOff = pIter->iLeafOffset; /* Offset to read at */ ASSERT_SZLEAF_OK(pIter->pLeaf); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); pIter->bDel = 0; pIter->nPos = 1; - if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + if( iOffpLeaf->p[iOff]==0 ){ pIter->bDel = 1; iOff++; - if( iOffpLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){ + if( iOffpLeaf->p[iOff]==0 ){ pIter->nPos = 1; iOff++; }else{ @@ -1643,13 +1644,16 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; - int nPos; - int bDummy; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ - + if( iiEndofDoclist = iOff; bNewTerm = 1; } - if( iOff>=pLeaf->szLeaf ){ + assert_nc( iOffszLeaf + || p->pConfig->eDetail==FTS5_DETAIL_NONE + ); + if( iOff>pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } @@ -1896,6 +1903,11 @@ static void fts5SegIterNext( #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } +#define fts5IndexSkipVarint(a, iOff) { \ + int iEnd = iOff+9; \ + while( (a[iOff++] & 0x80) && iOffiLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); } +#else + int iPoslist; + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + iPoslist = pIter->iTermLeafOffset; + }else{ + iPoslist = 4; + } + fts5IndexSkipVarint(pLeaf->p, iPoslist); + assert( eDetail==FTS5_DETAIL_NONE || iPoslist==( + pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) + )); + pIter->iLeafOffset = iPoslist; +#endif /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to @@ -2003,11 +2029,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#define fts5IndexSkipVarint(a, iOff) { \ - int iEnd = iOff+9; \ - while( (a[iOff++] & 0x80) && iOffpConfig->pgsz; - + int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -3870,12 +3891,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffn<=pBuf->nSpace ); - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); - }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffp[pBuf->n++] = 0; + iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - if( iPos>=nCopy ) break; } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + }else{ + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; } - iOff += nCopy; } } @@ -4408,16 +4442,18 @@ static void fts5MergeRowidLists( fts5NextRowid(p2, &i2, &iRowid2); while( i1>=0 || i2>=0 ){ if( i1>=0 && (i2<0 || iRowid1iOut ); fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); iOut = iRowid1; fts5NextRowid(p1, &i1, &iRowid1); }else{ + assert( iOut==0 || iRowid2>iOut ); fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); iOut = iRowid2; - fts5NextRowid(p2, &i2, &iRowid2); if( i1>=0 && iRowid1==iRowid2 ){ fts5NextRowid(p1, &i1, &iRowid1); } + fts5NextRowid(p2, &i2, &iRowid2); } } @@ -5217,7 +5253,7 @@ static int fts5QueryCksum( for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) - ){ + ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); @@ -5584,7 +5620,9 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); if( eDetail==FTS5_DETAIL_NONE ){ - cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); + } }else{ poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index cd43308513..95d72ebbc1 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -730,15 +730,18 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); - for(i=0; i<(pSorter->nIdx-1); i++){ - int iVal; - a += fts5GetVarint32(a, iVal); - iOff += iVal; - pSorter->aIdx[i] = iOff; + /* nBlob==0 in detail=none mode. */ + if( nBlob>0 ){ + for(i=0; i<(pSorter->nIdx-1); i++){ + int iVal; + a += fts5GetVarint32(a, iVal); + iOff += iVal; + pSorter->aIdx[i] = iOff; + } + pSorter->aIdx[i] = &aBlob[nBlob] - a; + pSorter->aPoslist = a; } - pSorter->aIdx[i] = &aBlob[nBlob] - a; - pSorter->aPoslist = a; fts5CsrNewrow(pCsr); } @@ -1678,15 +1681,15 @@ static int fts5ApiColumnText( } static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int n; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistWriter *aWriter; int i; -assert( pCsr->pSorter==0 ); /* fixme */ aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); if( aWriter==0 ) rc = SQLITE_NOMEM; for(i=0; inCol && rc==SQLITE_OK; i++){ @@ -1699,11 +1702,14 @@ assert( pCsr->pSorter==0 ); /* fixme */ } } sqlite3_free(aWriter); + if( pCsr->pSorter ){ + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); + } } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); n = pSorter->aIdx[iPhrase] - i1; @@ -2204,20 +2210,46 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: - /* Append the varints */ - for(i=0; i<(nPhrase-1); i++){ - const u8 *dummy; - int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); - sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); - } + /* Append the varints */ + for(i=0; i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } - /* Append the position lists */ - for(i=0; ipExpr, i, &pPoslist); - sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + /* Append the position lists */ + for(i=0; ipExpr, i, &pPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + case FTS5_DETAIL_COLUMNS: + + /* Append the varints */ + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ + const u8 *dummy; + int nByte; + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); + } + + /* Append the position lists */ + for(i=0; rc==SQLITE_OK && ipExpr, i, &pPoslist, &nPoslist); + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); + } + break; + + default: + break; } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 64af5c59b3..3bcd33dcfc 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -365,3 +365,30 @@ proc fts5_tokenize_split {text} { set ret } +#------------------------------------------------------------------------- +# +proc foreach_detail_mode {prefix script} { + set saved $::testprefix + foreach d [list full col none] { + set s [string map [list %DETAIL% $d] $script] + set ::detail $d + set ::testprefix "$prefix-$d" + reset_db + uplevel $s + unset ::detail + } + set ::testprefix $saved +} + +proc detail_check {} { + if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { + error "not in foreach_detail_mode {...} block" + } +} +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } +proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } + + + + diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 59b8ab86d1..e00bcfcdcc 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -21,6 +21,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; @@ -41,9 +43,9 @@ do_execsql_test 1.1 { #------------------------------------------------------------------------- # -reset_db + do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); @@ -66,11 +68,12 @@ do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); } + #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } foreach {i x y} { 1 {g f d b f} {h h e i a} @@ -93,7 +96,7 @@ foreach {i x y} { # reset_db do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -117,7 +120,7 @@ foreach {i x y} { # reset_db do_execsql_test 5.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -141,7 +144,7 @@ foreach {i x y} { # reset_db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -276,7 +279,7 @@ for {set i 1} {$i <= 10} {incr i} { # reset_db do_execsql_test 10.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); } set d10 { 1 {g f d b f} {h h e i a} @@ -309,19 +312,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { - CREATE VIRTUAL TABLE rank USING fts5(a, b, c); + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { - CREATE VIRTUAL TABLE t2 USING fts5(x,y); + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); } {} do_catchsql_test 12.2 { @@ -337,7 +340,7 @@ do_test 12.3 { # reset_db do_execsql_test 13.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} @@ -361,7 +364,7 @@ do_execsql_test 13.6 { # reset_db do_execsql_test 14.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' @@ -371,6 +374,10 @@ do_execsql_test 14.1 { INSERT INTO t1 SELECT * FROM d LIMIT 200; } +do_execsql_test 15.x { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + do_test 14.2 { set nRow 0 db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { @@ -433,7 +440,7 @@ do_catchsql_test 16.2 { # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE b2 USING fts5(x); + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); @@ -447,18 +454,20 @@ do_test 17.2 { set res } {{a b c} {a b c} {a b c}} -reset_db -do_execsql_test 18.1 { - CREATE VIRTUAL TABLE c2 USING fts5(x, y); - INSERT INTO c2 VALUES('x x x', 'x x x'); - SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; -} {1} +if {[string match n* %DETAIL%]==0} { + reset_db + do_execsql_test 17.3 { + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); + INSERT INTO c2 VALUES('x x x', 'x x x'); + SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; + } {1} +} #------------------------------------------------------------------------- # reset_db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE uio USING fts5(ttt); + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; @@ -505,8 +514,8 @@ do_execsql_test 17.9 { #-------------------------------------------------------------------- # do_execsql_test 18.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); - CREATE VIRTUAL TABLE t2 USING fts5(c, d); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } @@ -522,7 +531,7 @@ do_execsql_test 18.3 { # reset_db do_execsql_test 19.0 { - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; @@ -533,7 +542,7 @@ do_execsql_test 19.0 { # reset_db do_execsql_test 20.0 { - CREATE VIRTUAL TABLE temp.tmp USING fts5(x); + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] @@ -545,6 +554,7 @@ do_test 20.1 { execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } } $::ids +} finish_test diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 0746e64326..95da2cd2eb 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1 VALUES('hello', 'world'); INSERT INTO t1 VALUES('one two', 'three four'); INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); @@ -57,7 +59,7 @@ do_execsql_test 1.6 { reset_db do_execsql_test 2.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); INSERT INTO t1 VALUES('one'); INSERT INTO t1 VALUES('two'); @@ -159,7 +161,7 @@ foreach {tn expr res} { # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE s1 USING fts5(x); + CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); } foreach {tn doc} [list \ 1 [string repeat {a x } 1500000] \ @@ -172,8 +174,14 @@ do_execsql_test 4.3 { SELECT rowid FROM s1 WHERE s1 MATCH 'x' } {1 2} -do_execsql_test 4.4 { - SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' +if {[detail_is_full]} { + do_execsql_test 4.4 { + SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' + } {1 2} +} + +do_execsql_test 4.5 { + SELECT rowid FROM s1 WHERE s1 MATCH 'a x' } {1 2} #------------------------------------------------------------------------- @@ -182,7 +190,7 @@ do_execsql_test 4.4 { # (L-2) is larger than it. # do_execsql_test 5.0 { - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -222,7 +230,7 @@ do_test 5.2 { do_test 5.3 { execsql { DROP TABLE s2; - CREATE VIRTUAL TABLE s2 USING fts5(x); + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); INSERT INTO s2(s2, rank) VALUES('pgsz', 32); INSERT INTO s2(s2, rank) VALUES('automerge', 0); } @@ -241,7 +249,7 @@ do_test 5.4 { #------------------------------------------------------------------------- # do_execsql_test 6.0 { - CREATE VIRTUAL TABLE s3 USING fts5(x); + CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); BEGIN; INSERT INTO s3 VALUES('a b c'); INSERT INTO s3 VALUES('A B C'); @@ -276,13 +284,13 @@ do_test 6.4 { #------------------------------------------------------------------------- # set doc [string repeat "a b c " 500] -breakpoint do_execsql_test 7.0 { - CREATE VIRTUAL TABLE x1 USING fts5(x); + CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); INSERT INTO x1 VALUES($doc); } +} ;# foreach_detail_mode... finish_test diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index ff2affcce1..c3163147cb 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} @@ -205,8 +207,6 @@ proc collist_data {expr} { # End of test code #------------------------------------------------------------------------- -if 0 { - foreach {tn2 sql} { 1 {} 2 {BEGIN} @@ -215,7 +215,7 @@ foreach {tn2 sql} { fts5_aux_test_functions db do_execsql_test 1.$tn2.0 { - CREATE VIRTUAL TABLE xx USING fts5(x,y); + CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } @@ -228,31 +228,55 @@ foreach {tn2 sql} { execsql { INSERT INTO xx(xx) VALUES('integrity-check') } } {} - #------------------------------------------------------------------------- - # Test phrase queries. + # The following work with all detail= modes. # - foreach {tn phrase} { - 1 "o" - 2 "b q" - 3 "e a e" - 4 "m d g q q b k b w f q q p p" - 5 "l o o l v v k" - 6 "a" - 7 "b" - 8 "c" - 9 "no" - 10 "L O O L V V K" + foreach {tn expr} { + 1 "a AND b" + 2 "a OR b" + 3 "o" + 4 "b q" + 5 "e a e" + 6 "m d g q q b k b w f q q p p" + 7 "l o o l v v k" + 8 "a" + 9 "b" + 10 "c" + 11 "no" + 12 "L O O L V V K" + 13 "a AND b AND c" } { - set expr "\"$phrase\"" set res [poslist_data 1 $expr] - - do_execsql_test 1.$tn2.1.2.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.1.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.1.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.1.$tn.c.[llength $res] { + SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr + } $res + } + + if {[detail_is_none] || [detail_is_col]} continue + + #------------------------------------------------------------------------- + # Test phrase queries. + # + foreach {tn expr} { + 1 "b + q" + 2 "e + a + e" + 3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" + 4 "l + o + o + l + v + v + k" + 5 "L + O + O + L + V + V + K" + } { + set res [poslist_data 1 $expr] + + do_execsql_test 1.$tn2.2.$tn.p.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + + set res [collist_data $expr] + do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -261,25 +285,21 @@ foreach {tn2 sql} { # Test some AND and OR queries. # foreach {tn expr} { - 1.1 "a AND b" - 1.2 "a+b AND c" - 1.3 "d+c AND u" - 1.4 "d+c AND u+d" + 1.1 "a+b AND c" + 1.2 "d+c AND u" + 1.3 "d+c AND u+d" - 2.1 "a OR b" - 2.2 "a+b OR c" - 2.3 "d+c OR u" - 2.4 "d+c OR u+d" - - 3.1 { a AND b AND c } + 2.1 "a+b OR c" + 2.2 "d+c OR u" + 2.3 "d+c OR u+d" } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -308,12 +328,12 @@ foreach {tn2 sql} { 4.4 {{"y" y}:b} } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.3.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -334,12 +354,12 @@ foreach {tn2 sql} { 9 { y : NEAR(r c) } } { set res [poslist_data 1 $expr] - do_execsql_test 1.$tn2.4.1.$tn.p.[llength $res] { + do_execsql_test 1.$tn2.5.1.$tn.p.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res set res [collist_data $expr] - do_execsql_test 1.$tn2.4.1.$tn.c.[llength $res] { + do_execsql_test 1.$tn2.5.1.$tn.c.[llength $res] { SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr } $res } @@ -350,14 +370,14 @@ foreach {tn2 sql} { foreach {tn expr tclexpr} { 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} } { - do_execsql_test 1.$tn2.5.$tn { + do_execsql_test 1.$tn2.6.$tn { SELECT fts5_expr_tcl($expr, 'N $x') } [list $tclexpr] } #------------------------------------------------------------------------- # - do_execsql_test 1.$tn2.6.integrity { + do_execsql_test 1.$tn2.7.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} @@ -389,68 +409,18 @@ foreach {tn2 sql} { 19 { c NOT b OR a AND d } } { set res [poslist_data 0 $expr $bAsc] - do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res + do_execsql_test 1.$tn2.7.$bAsc.$tn.[llength $res] $sql $res } } } +} + do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} -} -#set data [lrange $data 0 5] - -# Some tests for detail=col tables and detail=none. -# -foreach {tn2 sql} { - 1 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); - } - 2 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); - BEGIN; - } - 3 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); - BEGIN; - } - 4 { - CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); - } -} { - reset_db - fts5_aux_test_functions db - - execsql $sql - - do_execsql_test 3.$tn2.0 { - INSERT INTO xx(xx, rank) VALUES('pgsz', 32); - } - - do_test 3.$tn2.1.1 { - foreach {id x y} $data { - execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } - } - execsql { INSERT INTO xx(xx) VALUES('integrity-check') } - } {} - - foreach {tn q} { - 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" - 11 "o a" 12 "c AND d" 13 "o OR a" 14 "c OR d" - } { - set res [poslist_data 1 $q] - do_execsql_test 3.$tn2.1.2.$tn.p.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q - } $res - - set res [collist_data $q] - do_execsql_test 3.$tn2.1.2.$tn.c.[llength $res] { - SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $q - } $res - } - -} +finish_test finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index cc86a6b2eb..b875354534 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -22,8 +22,12 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + +if {[detail_is_none]==0} continue + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE yy USING fts5(x, y); + CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); @@ -53,39 +57,23 @@ foreach {tn match res} { foreach {T create} { 2 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 3 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 4 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } 5 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - BEGIN; - } - - 6 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - } - - 7 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); - } - - 8 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } @@ -251,5 +239,7 @@ foreach {T create} { catchsql COMMIT } +} + finish_test diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index ded73d472f..5153306d19 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -22,8 +22,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -55,7 +57,7 @@ fts5_aux_test_functions db #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(x, y); + CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); } @@ -76,31 +78,35 @@ do_execsql_test 2.2 { 2 {1.0.2 1.0.10} } -do_execsql_test 2.3 { - SELECT rowid, fts5_test_poslist(t2) FROM t2 - WHERE t2 MATCH 'y:o' ORDER BY rowid; -} { - 1 {0.1.3 0.1.7} +if {[detail_is_full]} { + do_execsql_test 2.3 { + SELECT rowid, fts5_test_poslist(t2) FROM t2 + WHERE t2 MATCH 'y:o' ORDER BY rowid; + } { + 1 {0.1.3 0.1.7} + } } #------------------------------------------------------------------------- # do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t3 USING fts5(x, y); + CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); INSERT INTO t3 VALUES( 'r c', ''); } -do_execsql_test 3.1 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; -} { - 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} -} +if {[detail_is_full]} { + do_execsql_test 3.1 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; + } { + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} + } -do_execsql_test 3.2 { - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; -} { - 2 {0.0.0 1.0.1} + do_execsql_test 3.2 { + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; + } { + 2 {0.0.0 1.0.1} + } } do_execsql_test 3.3 { @@ -116,7 +122,7 @@ do_execsql_test 3.3 { #------------------------------------------------------------------------- # do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t4 USING fts5(x, y); + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t4 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); } @@ -134,7 +140,7 @@ reset_db fts5_aux_test_functions db do_execsql_test 5.1 { - CREATE VIRTUAL TABLE t5 USING fts5(x, y); + CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); INSERT INTO t5 VALUES('', 'a'); INSERT INTO t5 VALUES('a', ''); @@ -182,7 +188,7 @@ do_execsql_test 5.5 { reset_db fts5_aux_test_functions db do_execsql_test 6.1 { - CREATE VIRTUAL TABLE t6 USING fts5(x, y); + CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } @@ -200,7 +206,7 @@ do_execsql_test 6.2 { reset_db fts5_aux_test_functions db do_execsql_test 7.1 { - CREATE VIRTUAL TABLE t7 USING fts5(x, y); + CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); } do_test 7.2 { foreach {x y} { @@ -240,7 +246,7 @@ do_execsql_test 7.4 { #------------------------------------------------------------------------- # do_test 8.1 { - execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) } + execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } foreach {rowid x y} { 0 {A o} {o o o C o o o o o o o o} 1 {o o B} {o o o C C o o o o o o o} @@ -300,5 +306,7 @@ foreach {tn q cnt} { } $cnt } +} + finish_test diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index 8c50f84866..d6b2241568 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -24,9 +24,10 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); } proc do_snippet_test {tn doc match res} { @@ -111,34 +112,37 @@ foreach {tn doc res} { do_snippet_test 1.$tn $doc X $res } -foreach {tn doc res} { - 1.1 {X Y o o o o o} {[X Y] o o o o o} - 1.2 {o X Y o o o o} {o [X Y] o o o o} - 1.3 {o o X Y o o o} {o o [X Y] o o o} - 1.4 {o o o X Y o o} {o o o [X Y] o o} - 1.5 {o o o o X Y o} {o o o o [X Y] o} - 1.6 {o o o o o X Y} {o o o o o [X Y]} +if {[detail_is_full]} { + foreach {tn doc res} { + 1.1 {X Y o o o o o} {[X Y] o o o o o} + 1.2 {o X Y o o o o} {o [X Y] o o o o} + 1.3 {o o X Y o o o} {o o [X Y] o o o} + 1.4 {o o o X Y o o} {o o o [X Y] o o} + 1.5 {o o o o X Y o} {o o o o [X Y] o} + 1.6 {o o o o o X Y} {o o o o o [X Y]} - 2.1 {X Y o o o o o o} {[X Y] o o o o o...} - 2.2 {o X Y o o o o o} {o [X Y] o o o o...} - 2.3 {o o X Y o o o o} {o o [X Y] o o o...} - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} + 2.1 {X Y o o o o o o} {[X Y] o o o o o...} + 2.2 {o X Y o o o o o} {o [X Y] o o o o...} + 2.3 {o o X Y o o o o} {o o [X Y] o o o...} + 2.4 {o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o X Y} {...o o o o o [X Y]} - 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} - 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} - 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} - -} { - do_snippet_test 2.$tn $doc "X + Y" $res + 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} + 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} + 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} + 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} + } { + do_snippet_test 2.$tn $doc "X + Y" $res + } } +} ;# foreach_detail_mode + finish_test diff --git a/ext/fts5/test/fts5detail.test b/ext/fts5/test/fts5detail.test index 59f4b0e3c3..5bdd14424e 100644 --- a/ext/fts5/test/fts5detail.test +++ b/ext/fts5/test/fts5detail.test @@ -169,8 +169,8 @@ proc matchdata {expr {bAsc 1}} { foreach {tn tbl} { 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } + 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } } { -#break reset_db fts5_aux_test_functions db execsql $tbl @@ -198,7 +198,6 @@ foreach {tn tbl} { #------------------------------------------------------------------------- # Simple tests for detail=none tables. # -breakpoint do_execsql_test 4.0 { CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); @@ -209,5 +208,33 @@ do_catchsql_test 4.1 { SELECT * FROM t4('a:a') } {1 {fts5: column queries are not supported (detail=none)}} +#------------------------------------------------------------------------- +# Test that for the same content detail=none uses less space than +# detail=col, and that detail=col uses less space than detail=full +# +reset_db +do_test 5.1 { + foreach {tbl detail} {t1 none t2 col t3 full} { + execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" + foreach {rowid x y z} $::data { + execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" + } + } +} {} + +do_execsql_test 5.2 { + SELECT + (SELECT sum(length(block)) from t1_data) < + (SELECT sum(length(block)) from t2_data) +} {1} + +do_execsql_test 5.3 { + SELECT + (SELECT sum(length(block)) from t2_data) < + (SELECT sum(length(block)) from t3_data) +} {1} + + + finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index f99ea6112e..bb76250e9f 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -19,6 +19,8 @@ ifcapable !fts5 { return } +if 1 { + do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); INSERT INTO t1 VALUES('a b c'); @@ -163,5 +165,109 @@ do_execsql_test 9.2.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('b1'); + INSERT INTO t1 VALUES('b1'); + DELETE FROM t1 WHERE rowid=1; +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1('b1'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t1('b1') ORDER BY rowid DESC; +} {2} + +do_execsql_test 10.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} {} + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + WITH d(x,y) AS ( + SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d + ) + INSERT INTO t1 SELECT * FROM d LIMIT 23; +} + +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } +do_execsql_test 11.2 { + SELECT rowid FROM t1; +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.3 { + SELECT rowid FROM t1('xyz'); +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} + +do_execsql_test 11.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); + INSERT INTO yy VALUES('in if'); + INSERT INTO yy VALUES('if'); +} {} + +do_execsql_test 12.1 { + SELECT rowid FROM yy('i*'); +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); +} {} +foreach {rowid a} { + 0 {f} + 1 {u} + 2 {k} + 3 {a} + 4 {a} + 5 {u} + 6 {u} + 7 {u} + 8 {f} + 9 {f} + 10 {a} + 11 {p} + 12 {f} + 13 {u} + 14 {a} + 15 {a} +} { + do_execsql_test 13.1.$rowid { + INSERT INTO t1(rowid, a) VALUES($rowid, $a); + } +} + +} + +#------------------------------------------------------------------------- +# +reset_db +fts5_aux_test_functions db +do_execsql_test 14.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); + INSERT INTO t1 VALUES('a b c d'); +} {} + +do_execsql_test 14.1 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {0.0.1} + finish_test diff --git a/manifest b/manifest index c173c4014d..ce2b5eb30f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sfts5\sintegrity-check\scode\sso\sthat\sit\sworks\swith\sdetail=none\stables. -D 2015-12-31T18:39:14.423 +C Changes\sto\srun\smany\sfts5\stests\swith\sdetail=none\sand\sdetail=col\stables\sas\swell\sas\sthe\sdefault\sdetail=full.\sAlso\sfixes\sfor\sthe\sbugs\suncovered\sby\srunning\ssaid\stests. +D 2016-01-02T19:01:56.026 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,14 +97,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3 -F ext/fts5/fts5Int.h ab5f6f1f65652310a644d97d12aeabf8ab5ed17e +F ext/fts5/fts5Int.h cff7dd3131ba0db6f699df97237bf993f22c1a1f F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5 +F ext/fts5/fts5_expr.c 898e0bb53280b2c4c2702a2dc08dec731c7b32fe F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c a7cfe8e88621b473d749268f918b54b81a067c2e -F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89 +F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c +F ext/fts5/fts5_main.c f33439bde2e1023888e9b7f16e463e42fe4c00c5 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 @@ -114,13 +114,13 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl f923471d572d1e283b7d6985ce3cda62dae6fd45 -F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 -F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 1d2a10d4c13dcfd326287f51df3b7d90761c212c -F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3 -F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe -F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a +F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824 +F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc +F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b +F ext/fts5/test/fts5ac.test 9a3c5eb052a5e8c871d56d3d0f144e5fdcf075c2 +F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc +F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 +F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 @@ -139,7 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 -F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 +F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 @@ -170,7 +170,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456 -F ext/fts5/test/fts5simple2.test a1a9f9c443f8852f412ee9658ef667086ccb74d7 +F ext/fts5/test/fts5simple2.test d453e3ca100e08078a46b12547417891b326240a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca -R 70fdbbf97fa363362bfe1572d2790ec2 +P 3a1df69e58e5830da1dff158eedbe3817743f58f +R 584d5f1c8b7a15ac2eae2ab1ffe71c4a U dan -Z 9907fb7300f9e53b05e209bde8721940 +Z cff9ad15204445a84003e93595871343 diff --git a/manifest.uuid b/manifest.uuid index 7093688530..4a0bbbdcd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a1df69e58e5830da1dff158eedbe3817743f58f \ No newline at end of file +6322a1d984e7946735bace8a069ef24b31754b3b \ No newline at end of file