From 882ef0b8c08fd475df3785b37094b00b4173eeb2 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Aug 2016 19:26:57 +0000 Subject: [PATCH] Have fts5 interpret column lists that begin with a "-" character as "match any column except" lists. FossilOrigin-Name: e517545650631d1e8a7ee63c6646a8b183a0a894 --- ext/fts5/fts5Int.h | 1 + ext/fts5/fts5_expr.c | 61 +++++++++++++++++++++++++++++++++++ ext/fts5/fts5_index.c | 12 +++++++ ext/fts5/fts5parse.y | 11 ++++++- ext/fts5/test/fts5colset.test | 59 +++++++++++++++++++++++++++++++++ manifest | 24 +++++++------- manifest.uuid | 2 +- 7 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 ext/fts5/test/fts5colset.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c7169a113b..9b1316d454 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -737,6 +737,7 @@ void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +void sqlite3Fts5ParseColsetNegative(Fts5Parse*, int); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9119813c85..be6d768a68 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -124,6 +124,7 @@ struct Fts5Parse { char *zErr; int rc; int nPhrase; /* Size of apPhrase array */ + int bNegativeCollist; /* Column list being parsed started with - */ Fts5ExprPhrase **apPhrase; /* Array of all phrases */ Fts5ExprNode *pExpr; /* Result of a successful parse */ }; @@ -167,6 +168,7 @@ static int fts5ExprGetToken( case ',': tok = FTS5_COMMA; break; case '+': tok = FTS5_PLUS; break; case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; case '\0': tok = FTS5_EOF; break; case '"': { @@ -1793,6 +1795,53 @@ static Fts5Colset *fts5ParseColset( return pNew; } +/* +** The second argument passed to this function may be NULL, or it may be +** an existing Fts5Colset object. If it is passed NULL, this function +** returns a pointer to a new Fts5Colset object containing entries for +** all table columns except column iCol. If an OOM error occurs trying to +** allocate the Fts5Colset object, an error code is stored in pParse and +** NULL returned. +** +** If the second argument is not NULL, a copy of it is returned. Before +** returning, any entry for column iCol is removed. It is not an error +** if the Fts5Colset object does not contain an entry for column iCol +** when this function is called. +*/ +static Fts5Colset *fts5ParseNegativeColset( + Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ + Fts5Colset *p, /* Existing colset object */ + int iCol /* New column to add to colset object */ +){ + int i; + Fts5Colset *pRet = p; + + if( pRet==0 ){ + int nCol = pParse->pConfig->nCol; + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet==0 ) return 0; + pRet->nCol = nCol; + for(i=0; iaiCol[i] = i; + } + } + + for(i=0; inCol; i++){ + if( pRet->aiCol[i]==iCol ){ + int nByte = sizeof(int)*(pRet->nCol-i-1); + if( nByte ){ + memmove(&pRet->aiCol[i], &pRet->aiCol[i+1], nByte); + } + pRet->nCol--; + break; + } + } + + return pRet; +} + Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ Fts5Colset *pColset, /* Existing colset object */ @@ -1811,6 +1860,8 @@ Fts5Colset *sqlite3Fts5ParseColset( } if( iCol==pConfig->nCol ){ sqlite3Fts5ParseError(pParse, "no such column: %s", z); + }else if( pParse->bNegativeCollist ){ + pRet = fts5ParseNegativeColset(pParse, pColset, iCol); }else{ pRet = fts5ParseColset(pParse, pColset, iCol); } @@ -1825,6 +1876,16 @@ Fts5Colset *sqlite3Fts5ParseColset( return pRet; } +/* +** Set (bVal==1) or clear (bVal==0) the Fts5Parse.bNegativeCollist flag. +** +** The parser calls this function as it begins to parse a colset (Fts5Colset +** object) with bVal set to 1 if the colset begins with a "-" or 0 otherwise. +*/ +void sqlite3Fts5ParseColsetNegative(Fts5Parse *pParse, int bVal){ + pParse->bNegativeCollist = bVal; +} + void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 1d5ebe1428..52fdc634a5 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3195,6 +3195,14 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.nData = 0; +} + /* ** xSetOutputs callback used by detail=col when there is a column filter ** and there are 100 or more columns. Also called as a fallback from @@ -3300,6 +3308,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y index 1607d3846a..8bc95f7364 100644 --- a/ext/fts5/fts5parse.y +++ b/ext/fts5/fts5parse.y @@ -119,18 +119,27 @@ cnearset(A) ::= colset(X) COLON nearset(Y). { %destructor colset { sqlite3_free($$); } %type colsetlist {Fts5Colset*} %destructor colsetlist { sqlite3_free($$); } +%type minus_opt {int} colset(A) ::= LCP colsetlist(X) RCP. { A = X; } +colset(A) ::= MINUS STRING(X). { + sqlite3Fts5ParseColsetNegative(pParse, 1); + A = sqlite3Fts5ParseColset(pParse, 0, &X); +} colset(A) ::= STRING(X). { + sqlite3Fts5ParseColsetNegative(pParse, 0); A = sqlite3Fts5ParseColset(pParse, 0, &X); } colsetlist(A) ::= colsetlist(Y) STRING(X). { A = sqlite3Fts5ParseColset(pParse, Y, &X); } -colsetlist(A) ::= STRING(X). { +colsetlist(A) ::= minus_opt(M) STRING(X). { + sqlite3Fts5ParseColsetNegative(pParse, M); A = sqlite3Fts5ParseColset(pParse, 0, &X); } +minus_opt(A) ::= MINUS. { A = 1; } +minus_opt(A) ::= . { A = 0; } %type nearset {Fts5ExprNearset*} %type nearphrases {Fts5ExprNearset*} diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test new file mode 100644 index 0000000000..c09974a742 --- /dev/null +++ b/ext/fts5/test/fts5colset.test @@ -0,0 +1,59 @@ +# 2016 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5colset + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $::testprefix { + if {[detail_is_none]} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 + } + + foreach {tn q res} { + 1 "a" {1 2 3 4} + 2 "{a} : a" {1} + 3 "{-a} : a" {2 3 4} + 4 "{-a c} : a" {2 4} + 5 "{-d d c} : a" {1 2} + 6 "{-d c b a} : a" {} + 7 "{-\"a\"} : b" {1 2 3} + 8 "- c : a" {1 2 4} + 9 "-c : a" {1 2 4} + 10 "-\"c\" : a" {1 2 4} + } { + breakpoint + do_execsql_test 1.$tn { + SELECT rowid FROM t1($q) + } $res + } + + +} + + +finish_test + + diff --git a/manifest b/manifest index 9d5c14e49e..df25caa15d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.14 -D 2016-08-08T13:40:27.974 +C Have\sfts5\sinterpret\scolumn\slists\sthat\sbegin\swith\sa\s"-"\scharacter\sas\s"match\sany\scolumn\sexcept"\slists. +D 2016-08-09T19:26:57.822 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -98,13 +98,13 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 -F ext/fts5/fts5Int.h 9bd0c7c64285b5b368eca0ac63613185c5ad24ba +F ext/fts5/fts5Int.h 51eb867d9afbd1a4130fde00c39acf9aacabe1b6 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c bcb238ee4ac1164302ab528487520488516bd030 +F ext/fts5/fts5_expr.c 8e975ae07dbff244adea3a3697f027fa5387a991 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 -F ext/fts5/fts5_index.c b429e23fabb57506f71e406997cc46b89190dc97 +F ext/fts5/fts5_index.c e25ac419fc66f412e6044595b20b4bf8f7cea284 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 F ext/fts5/fts5_storage.c de0ed8a06738bde433afe11e92295ceaffbc4e58 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a @@ -114,7 +114,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c dba72ca393d71c2588548b51380387f6b44c77a8 -F ext/fts5/fts5parse.y fcc5e92e570d38cab38488b2109cbf67468923b2 +F ext/fts5/fts5parse.y bc2f2d9a726e69443ca58a5c0164283a63da819e F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 F ext/fts5/test/fts5aa.test bd2d88182b9f7f30d300044048ad14683306b745 @@ -135,6 +135,7 @@ F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a +F ext/fts5/test/fts5colset.test ad686cc648264f0334d2cc11b842e99f6e2bc10a F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5 @@ -1509,10 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebc396a19fa79bea208ecda277ffff5d02166d0b -R e7962d3372d6a1e29e3980753a6f0162 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.14.0 * -U drh -Z ecb9b7072c2b71d36f58909f83563ec2 +P d5e98057028abcf7217d0d2b2e29bbbcdf09d6de +R bd788c6a4ffbd14fc48e3960d75681eb +U dan +Z 8eb6c4d9289f254e7f02b4786ea43ed2 diff --git a/manifest.uuid b/manifest.uuid index 732a4af61b..0c0e3e4b5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5e98057028abcf7217d0d2b2e29bbbcdf09d6de \ No newline at end of file +e517545650631d1e8a7ee63c6646a8b183a0a894 \ No newline at end of file