Fix another problem with fts3/4 auxiliary functions and NEAR expressions that consist entirely of deferred tokens.

FossilOrigin-Name: a8c91c132f6157b7e3649f57a799984b1d7f8a18fd434515c875617d4195db29

ext/fts3/fts3.c

@@ -5765,6 +5765,21 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
   }
 }
 
+/*
+** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array
+** has not yet been allocated, allocate and zero it. Otherwise, just zero
+** it.
+*/
+static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){
+  Fts3Table *pTab = (Fts3Table*)pCtx;
+  if( pExpr->aMI==0 ){
+    pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
+    if( pExpr->aMI==0 ) return SQLITE_NOMEM;
+  }
+  memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
+  return SQLITE_OK;
+}
+
 /*
 ** Expression pExpr must be of type FTSQUERY_PHRASE.
 **
@@ -5786,7 +5801,6 @@ static int fts3EvalGatherStats(
   if( pExpr->aMI==0 ){
     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
     Fts3Expr *pRoot;                /* Root of NEAR expression */
-    Fts3Expr *p;                    /* Iterator used for several purposes */
 
     sqlite3_int64 iPrevId = pCsr->iPrevId;
     sqlite3_int64 iDocid;
@@ -5794,7 +5808,9 @@ static int fts3EvalGatherStats(
 
     /* Find the root of the NEAR expression */
     pRoot = pExpr;
-    while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
+    while( pRoot->pParent 
+        && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) 
+    ){
       pRoot = pRoot->pParent;
     }
     iDocid = pRoot->iDocid;
@@ -5802,14 +5818,8 @@ static int fts3EvalGatherStats(
     assert( pRoot->bStart );
 
     /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
-    for(p=pRoot; p; p=p->pLeft){
+    rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab);
-      Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
+    if( rc!=SQLITE_OK ) return rc;
-      assert( pE->aMI==0 );
-      pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
-      if( !pE->aMI ) return SQLITE_NOMEM;
-      memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
-    }
-
     fts3EvalRestart(pCsr, pRoot, &rc);
 
     while( pCsr->isEof==0 && rc==SQLITE_OK ){
@@ -5965,6 +5975,7 @@ int sqlite3Fts3EvalPhrasePoslist(
     u8 bTreeEof = 0;
     Fts3Expr *p;                  /* Used to iterate from pExpr to root */
     Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
+    Fts3Expr *pRun;               /* Closest non-deferred ancestor of pNear */
     int bMatch;
 
     /* Check if this phrase descends from an OR expression node. If not, 
@@ -5979,25 +5990,30 @@ int sqlite3Fts3EvalPhrasePoslist(
       if( p->bEof ) bTreeEof = 1;
     }
     if( bOr==0 ) return SQLITE_OK;
+    pRun = pNear;
+    while( pRun->bDeferred ){
+      assert( pRun->pParent );
+      pRun = pRun->pParent;
+    }
 
     /* This is the descendent of an OR node. In this case we cannot use
     ** an incremental phrase. Load the entire doclist for the phrase
     ** into memory in this case.  */
     if( pPhrase->bIncr ){
-      int bEofSave = pNear->bEof;
+      int bEofSave = pRun->bEof;
-      fts3EvalRestart(pCsr, pNear, &rc);
+      fts3EvalRestart(pCsr, pRun, &rc);
-      while( rc==SQLITE_OK && !pNear->bEof ){
+      while( rc==SQLITE_OK && !pRun->bEof ){
-        fts3EvalNextRow(pCsr, pNear, &rc);
+        fts3EvalNextRow(pCsr, pRun, &rc);
-        if( bEofSave==0 && pNear->iDocid==iDocid ) break;
+        if( bEofSave==0 && pRun->iDocid==iDocid ) break;
       }
       assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
-      if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){
+      if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){
         rc = FTS_CORRUPT_VTAB;
       }
     }
     if( bTreeEof ){
-      while( rc==SQLITE_OK && !pNear->bEof && !pNear->bDeferred ){
+      while( rc==SQLITE_OK && !pRun->bEof ){
-        fts3EvalNextRow(pCsr, pNear, &rc);
+        fts3EvalNextRow(pCsr, pRun, &rc);
       }
     }
     if( rc!=SQLITE_OK ) return rc;

ext/fts3/fts3Int.h

@@ -650,5 +650,7 @@ int sqlite3FtsUnicodeIsalnum(int);
 int sqlite3FtsUnicodeIsdiacritic(int);
 #endif
 
+int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
+
 #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
 #endif /* _FTSINT_H */

ext/fts3/fts3_snippet.c

@@ -41,7 +41,7 @@ typedef sqlite3_int64 i64;
 
 
 /*
-** Used as an fts3ExprIterate() context when loading phrase doclists to
+** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to
 ** Fts3Expr.aDoclist[]/nDoclist.
 */
 typedef struct LoadDoclistCtx LoadDoclistCtx;
@@ -85,7 +85,7 @@ struct SnippetFragment {
 };
 
 /*
-** This type is used as an fts3ExprIterate() context object while 
+** This type is used as an sqlite3Fts3ExprIterate() context object while 
 ** accumulating the data returned by the matchinfo() function.
 */
 typedef struct MatchInfo MatchInfo;
@@ -244,7 +244,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){
 }
 
 /*
-** Helper function for fts3ExprIterate() (see below).
+** Helper function for sqlite3Fts3ExprIterate() (see below).
 */
 static int fts3ExprIterate2(
   Fts3Expr *pExpr,                /* Expression to iterate phrases of */
@@ -278,7 +278,7 @@ static int fts3ExprIterate2(
 ** Otherwise, SQLITE_OK is returned after a callback has been made for
 ** all eligible phrase nodes.
 */
-static int fts3ExprIterate(
+int sqlite3Fts3ExprIterate(
   Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   void *pCtx                      /* Second argument to pass to callback */
@@ -287,10 +287,9 @@ static int fts3ExprIterate(
   return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
 }
 
-
 /*
-** This is an fts3ExprIterate() callback used while loading the doclists
+** This is an sqlite3Fts3ExprIterate() callback used while loading the 
-** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
+** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
 ** fts3ExprLoadDoclists().
 */
 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
@@ -322,9 +321,9 @@ static int fts3ExprLoadDoclists(
   int *pnToken                    /* OUT: Number of tokens in query */
 ){
   int rc;                         /* Return Code */
-  LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
+  LoadDoclistCtx sCtx = {0,0,0};  /* Context for sqlite3Fts3ExprIterate() */
   sCtx.pCsr = pCsr;
-  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
+  rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx);
   if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   if( pnToken ) *pnToken = sCtx.nToken;
   return rc;
@@ -337,7 +336,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
 }
 static int fts3ExprPhraseCount(Fts3Expr *pExpr){
   int nPhrase = 0;
-  (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
+  (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
   return nPhrase;
 }
 
@@ -465,8 +464,9 @@ static void fts3SnippetDetails(
 }
 
 /*
-** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
+** This function is an sqlite3Fts3ExprIterate() callback used by
-** Each invocation populates an element of the SnippetIter.aPhrase[] array.
+** fts3BestSnippet().  Each invocation populates an element of the
+** SnippetIter.aPhrase[] array.
 */
 static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
   SnippetIter *p = (SnippetIter *)ctx;
@@ -556,7 +556,9 @@ static int fts3BestSnippet(
   sIter.nSnippet = nSnippet;
   sIter.nPhrase = nList;
   sIter.iCurrent = -1;
-  rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
+  rc = sqlite3Fts3ExprIterate(
+      pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter
+  );
   if( rc==SQLITE_OK ){
 
     /* Set the *pmSeen output variable. */
@@ -917,10 +919,10 @@ static int fts3ExprLHitGather(
 }
 
 /*
-** fts3ExprIterate() callback used to collect the "global" matchinfo stats
+** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo
-** for a single query. 
+** stats for a single query. 
 **
-** fts3ExprIterate() callback to load the 'global' elements of a
+** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a
 ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
 ** of the matchinfo array that are constant for all rows returned by the 
 ** current query.
@@ -955,7 +957,7 @@ static int fts3ExprGlobalHitsCb(
 }
 
 /*
-** fts3ExprIterate() callback used to collect the "local" part of the
+** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the
 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
 ** array that are different for each row returned by the query.
 */
@@ -1151,7 +1153,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
   **/
   aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase);
   if( !aIter ) return SQLITE_NOMEM;
-  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+  (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
 
   for(i=0; i<pInfo->nPhrase; i++){
     LcsIterator *pIter = &aIter[i];
@@ -1328,11 +1330,11 @@ static int fts3MatchinfoValues(
             rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
             if( rc!=SQLITE_OK ) break;
           }
-          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+          rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
           sqlite3Fts3EvalTestDeferred(pCsr, &rc);
           if( rc!=SQLITE_OK ) break;
         }
-        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
+        (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
         break;
       }
     }
@@ -1555,7 +1557,7 @@ struct TermOffsetCtx {
 };
 
 /*
-** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
+** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets().
 */
 static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
   TermOffsetCtx *p = (TermOffsetCtx *)ctx;
@@ -1637,7 +1639,9 @@ void sqlite3Fts3Offsets(
     */
     sCtx.iCol = iCol;
     sCtx.iTerm = 0;
-    rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
+    rc = sqlite3Fts3ExprIterate(
+        pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx
+    );
     if( rc!=SQLITE_OK ) goto offsets_out;
 
     /* Retreive the text stored in column iCol. If an SQL NULL is stored 

manifest

@@ -1,5 +1,5 @@
-C Add\sscalar\sSQL\sfunction\sunhex().
+C Fix\sanother\sproblem\swith\sfts3/4\sauxiliary\sfunctions\sand\sNEAR\sexpressions\sthat\sconsist\sentirely\sof\sdeferred\stokens.
-D 2023-01-24T20:17:43.328
+D 2023-01-25T13:42:55.473
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -58,16 +58,16 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 36e4555d96d6d322d46d8bbf5a2fe30422be2071d160ebeb3e2156b0ce085b91
+F ext/fts3/fts3.c a613b81b208a330ce155b5f41ee038358aa4ddbae8b12fe9d07d7b399bf57dc8
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h ae2a44b04cddb5fb35cac4ea5f7f819b2894fd258186465777a19f7acfdf84ed
+F ext/fts3/fts3Int.h e573c6d881f7238d77cc3fd2396cbb9b2fe13efef7d2ad295a155151c4e7efbd
 F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f
 F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
 F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8
-F ext/fts3/fts3_snippet.c f9a8149173553113f3c495a503843e30028b5dc3723d0ca798c5ad6142e130e6
+F ext/fts3/fts3_snippet.c 4d6523e3eddeb7b46e7a82b3476a0a86a0c04821e0e2b8dd40f45ee28057cb13
 F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1
 F ext/fts3/fts3_test.c d8d7b2734f894e8a489987447658e374cdd3a3bc8575c401decf1911cb7c6454
 F ext/fts3/fts3_tokenize_vtab.c a95feda3590f3c3e17672fe35b67ea6112471aeea4c07ef7744a6606b66549aa
@@ -1067,7 +1067,7 @@ F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05ee
 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf
 F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f
 F test/fts3defer.test f4c20e4c7153d20a98ee49ee5f3faef624fefc9a067f8d8d629db380c4d9f1de
-F test/fts3defer2.test 0c0448ec227f470250173dd41b94fa4776d7dcaee07a7b8c0f7e6c1aba97f024
+F test/fts3defer2.test 356c980bc28cdf4b14dbd3d7599929d3e54ecf9a37760c037bb895f614918459
 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
 F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
 F test/fts3dropmod.test 7de242ea1c8a713a8b143ea54468f4b1c4953fa068349e23ac178e2c90c59889
@@ -2044,9 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39bfae4c4698a13e07c4a0725f2790955e03b601fe64e17a000c691def1bdcb8 66c8562690b19f17972589611810e1dccad3a48777acb05208289c1f77076f71
+P 890e9629a7480138c9c1d3acc2d1e7b3c05e0d156e5c5fba428bc1aeb790fbfb
-R e4f755d732b8075e7847f3615f7f161a
+R 1da822a94c4b832d6a6cd41c425e2e73
-T +closed 66c8562690b19f17972589611810e1dccad3a48777acb05208289c1f77076f71
 U dan
-Z b2c1f41f5ac4263ca444db1f00dc6050
+Z 20acb0606f0068559b373b7ba0fbeacf
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-890e9629a7480138c9c1d3acc2d1e7b3c05e0d156e5c5fba428bc1aeb790fbfb
+a8c91c132f6157b7e3649f57a799984b1d7f8a18fd434515c875617d4195db29

test/fts3defer2.test

@@ -164,15 +164,25 @@ foreach {tn sql} {
 do_execsql_test 2.5 {
   INSERT INTO t3(t3) VALUES('rebuild');
 }
-do_execsql_test 2.5 {
+do_execsql_test 2.6 {
   SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR a)';
 } {11 228929}
-do_execsql_test 2.6 {
+do_execsql_test 2.7 {
   SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b NEAR a)';
 } {1 23 3 23 11 205}
-do_execsql_test 2.7 {
+do_execsql_test 2.8 {
   SELECT rowid, length(offsets(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b)';
 } {1 15 3 15 11 106}
 
+do_execsql_test 2.9 {
+  SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR a)';
+} {11 32}
+do_execsql_test 2.10 {
+  SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b NEAR a)'
+} {1 44 3 44 11 44}
+do_execsql_test 2.11 {
+  SELECT rowid, length(matchinfo(t3)) FROM t3 WHERE t3 MATCH '(a NEAR b)';
+} {1 32 3 32 11 32}
+
 
 finish_test