Defer building xInstToken() hash-table until it is to be used.

FossilOrigin-Name: 9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa

ext/fts5/fts5Int.h

@@ -550,7 +550,6 @@ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
 int sqlite3Fts5IndexIterWriteTokendata(
     Fts5IndexIter*, const char*, int, int iCol, int iOff
 );
-int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter*);
 
 /*
 ** End of interface to code in fts5_index.c.

ext/fts5/fts5_expr.c

@@ -3044,7 +3044,6 @@ int sqlite3Fts5ExprPopulatePoslists(
   const char *z, int n
 ){
   int i;
-  int rc = SQLITE_OK;
   Fts5ExprCtx sCtx;
   sCtx.pExpr = pExpr;
   sCtx.aPopulator = aPopulator;
@@ -3073,20 +3072,9 @@ int sqlite3Fts5ExprPopulatePoslists(
     }
   }
 
-  rc = sqlite3Fts5Tokenize(pConfig, 
+  return sqlite3Fts5Tokenize(pConfig, 
       FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
   );
-
-  if( pConfig->bTokendata ){
-    for(i=0; i<pExpr->nPhrase; i++){
-      Fts5ExprTerm *pT;
-      for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
-        sqlite3Fts5IndexIterHashifyTokendata(pT->pIter);
-      }
-    }
-  }
-
-  return rc;
 }
 
 static void fts5ExprClearPoslists(Fts5ExprNode *pNode){

ext/fts5/fts5_index.c

@@ -628,6 +628,8 @@ struct Fts5TokenMapToken {
 };
 
 struct Fts5TokenMap {
+  int bHashed;                    /* True once hashed */
+
   int nEntryAlloc;
   int nEntry;
   Fts5TokenMapEntry *aEntry;
@@ -6372,7 +6374,6 @@ static void fts5SetupPrefixIter(
     ** index contains all the doclists required, except for the one
     ** corresponding to the prefix itself. That one is extracted from the
     ** main term index here.  */
-    assert( iIdx==0 || pMap==0 );
     if( iIdx!=0 ){
       int dummy = 0;
       const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT;
@@ -6402,7 +6403,7 @@ static void fts5SetupPrefixIter(
       *ppIter = p1;
     }else{
 
-      if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL ){
+      if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL && bTokenscan ){
         pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap));
       }
       assert( p->rc!=SQLITE_OK || (aBuf && pStruct) );
@@ -6473,7 +6474,6 @@ static void fts5SetupPrefixIter(
         pData->p = (u8*)&pData[1];
         pData->nn = pData->szLeaf = doclist.n;
         if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n);
-        if( pMap ) fts5TokenMapHashify(p, pMap);
         fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter);
         pMap = 0;
       }
@@ -6867,7 +6867,15 @@ int sqlite3Fts5IterToken(
   const char **ppOut, int *pnOut
 ){
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter->pTokenMap ){
+  Fts5TokenMap *pMap = pIter->pTokenMap;
+  if( pMap ){
+    if( pMap->bHashed==0 ){
+      Fts5Index *p = pIter->pIndex;
+      fts5TokenMapHashify(p, pMap);
+      if( p->rc ){
+        return fts5IndexReturn(p);
+      }
+    }
     *ppOut = (const char*)fts5TokenMapLookup(
         pIter->pTokenMap, pIndexIter->iRowid, iCol, iOff, pnOut
     );
@@ -6936,14 +6944,6 @@ int sqlite3Fts5IndexIterWriteTokendata(
   return fts5IndexReturn(p);
 }
 
-int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter *pIndexIter){
-  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter->pTokenMap ){
-    fts5TokenMapHashify(pIter->pIndex, pIter->pTokenMap);
-  }
-  return fts5IndexReturn(pIter->pIndex);
-}
-
 /*
 ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
 */

manifest

@@ -1,5 +1,5 @@
-C Fix\stokendata=1\sand\sxInstToken()\sAPIs\sfor\sdetail=none\sand\sdetail=column\stables.
+C Defer\sbuilding\sxInstToken()\shash-table\suntil\sit\sis\sto\sbe\sused.
-D 2023-11-22T19:02:54.078
+D 2023-11-22T20:02:55.862
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -88,13 +88,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c
-F ext/fts5/fts5Int.h d330c2e20051c300b26325b8ba29aa89e99d301c80e2f51092e5bb70346a17cd
+F ext/fts5/fts5Int.h 782151060d176be22861f57bf38e087a82cfb0dfc4b2fa6f9ccbc2641b6d01e3
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c 0d846134eafeeb1f0724b9c8cc02a2ef9c4082519aa3923173deadd5155910b1
+F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 7b87808d788238eff4a0a68728e6ed49817e71bbfb328a18050d7d8e92a5d66a
+F ext/fts5/fts5_index.c 710b022dcdf152eb7bbbc3f83eb662e1e67c25e0643416096ed070b10d7829fb
 F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
@@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47
+P 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86
-R 161c23f360cac1706a2c5f6b11155312
+R 0d85011f1dd994f228f63f6f67d53fdc
 U dan
-Z 281a4d74e3cce55af028079774718a8b
+Z f3ab4e11005318d9a7fa0f164d5ead9f
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86
+9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa