Fix tokendata=1 and xInstToken() APIs for detail=none and detail=column tables.

FossilOrigin-Name: 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86

ext/fts5/fts5Int.h

@@ -545,6 +545,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
 
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+int sqlite3Fts5IndexIterWriteTokendata(
+    Fts5IndexIter*, const char*, int, int iCol, int iOff
+);
+int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter*);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/

ext/fts5/fts5_expr.c

@@ -2985,6 +2985,12 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
   return 0;
 }
 
+static int fts5QueryTerm(const char *pToken, int nToken){
+  int ii;
+  for(ii=0; ii<nToken && pToken[ii]; ii++){}
+  return ii;
+}
+
 static int fts5ExprPopulatePoslistsCb(
   void *pCtx,                /* Copy of 2nd argument to xTokenize() */
   int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2996,21 +3002,32 @@ static int fts5ExprPopulatePoslistsCb(
   Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
   Fts5Expr *pExpr = p->pExpr;
   int i;
+  int nQuery = nToken;
 
   UNUSED_PARAM2(iUnused1, iUnused2);
 
-  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+  if( pExpr->pConfig->bTokendata ){
+    nQuery = fts5QueryTerm(pToken, nQuery);
+  }
   if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
   for(i=0; i<pExpr->nPhrase; i++){
     Fts5ExprTerm *pT;
     if( p->aPopulator[i].bOk==0 ) continue;
     for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
-      if( (pT->nFullTerm==nToken || (pT->nFullTerm<nToken && pT->bPrefix))
-       && memcmp(pT->pTerm, pToken, pT->nFullTerm)==0
+      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
       ){
         int rc = sqlite3Fts5PoslistWriterAppend(
             &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         );
+        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){
+          int iCol = p->iOff>>32;
+          int iTokOff = p->iOff & 0x7FFFFFFF;
+          rc = sqlite3Fts5IndexIterWriteTokendata(
+              pT->pIter, pToken, nToken, iCol, iTokOff
+          );
+        }
         if( rc ) return rc;
         break;
       }
@@ -3027,11 +3044,23 @@ int sqlite3Fts5ExprPopulatePoslists(
   const char *z, int n
 ){
   int i;
+  int rc = SQLITE_OK;
   Fts5ExprCtx sCtx;
   sCtx.pExpr = pExpr;
   sCtx.aPopulator = aPopulator;
   sCtx.iOff = (((i64)iCol) << 32) - 1;
 
+  /* If this is a tokendata=1 table, clear out the hash tables of
+  ** full-terms.  */
+  if( pConfig->bTokendata ){
+    for(i=0; i<pExpr->nPhrase; i++){
+      Fts5ExprTerm *pT;
+      for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+        sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+      }
+    }
+  }
+
   for(i=0; i<pExpr->nPhrase; i++){
     Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
     Fts5Colset *pColset = pNode->pNear->pColset;
@@ -3044,9 +3073,20 @@ int sqlite3Fts5ExprPopulatePoslists(
     }
   }
 
-  return sqlite3Fts5Tokenize(pConfig, 
+  rc = sqlite3Fts5Tokenize(pConfig, 
       FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
   );
+
+  if( pConfig->bTokendata ){
+    for(i=0; i<pExpr->nPhrase; i++){
+      Fts5ExprTerm *pT;
+      for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+        sqlite3Fts5IndexIterHashifyTokendata(pT->pIter);
+      }
+    }
+  }
+
+  return rc;
 }
 
 static void fts5ExprClearPoslists(Fts5ExprNode *pNode){

ext/fts5/fts5_index.c

@@ -6877,6 +6877,73 @@ int sqlite3Fts5IterToken(
   return SQLITE_OK;
 }
 
+/*
+** Clear any existing entries from the token-map associated with the
+** iterator passed as the only argument. 
+*/
+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+  assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL );
+  if( pIter->pTokenMap ){
+    pIter->pTokenMap->nEntry = 0;
+  }
+}
+
+int sqlite3Fts5IndexIterWriteTokendata(
+  Fts5IndexIter *pIndexIter, 
+  const char *pToken, int nToken, 
+  int iCol, int iOff
+){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+  Fts5Index *p = pIter->pIndex;
+  assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
+  if( pIter->pTokenMap==0 ){
+    pIter->pTokenMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap));
+  }
+  if( p->rc==SQLITE_OK ){
+    Fts5TokenMap *pMap = pIter->pTokenMap;
+    int ii;
+    for(ii=0; ii<pMap->nToken; ii++){
+      if( nToken==pMap->aToken[ii].nTerm 
+       && 0==memcmp(pMap->aToken[ii].pTerm, pToken, nToken) 
+      ){
+        break;
+      }
+    }
+    if( ii==pMap->nToken ){
+      fts5TokenMapTerm(p, pMap, (const u8*)pToken, nToken);
+    }
+    if( pMap->nEntry>=pMap->nEntryAlloc ){
+      int nNew = pMap->nEntryAlloc ? pMap->nEntryAlloc*2 : 32;
+      Fts5TokenMapEntry *aNew = (Fts5TokenMapEntry*)sqlite3_realloc(
+          pMap->aEntry, nNew * sizeof(Fts5TokenMapEntry)
+      );
+      if( aNew==0 ){
+        p->rc = SQLITE_NOMEM;
+      }else{
+        pMap->aEntry = aNew;
+        pMap->nEntryAlloc = nNew;
+      }
+    }
+    if( p->rc==SQLITE_OK ){
+      Fts5TokenMapEntry *pEntry = &pMap->aEntry[pMap->nEntry++];
+      pEntry->iRowid = pIndexIter->iRowid;
+      pEntry->iCol = iCol;
+      pEntry->iOff = iOff;
+      pEntry->iTok = ii+1;
+    }
+  }
+  return fts5IndexReturn(p);
+}
+
+int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter *pIndexIter){
+  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+  if( pIter->pTokenMap ){
+    fts5TokenMapHashify(pIter->pIndex, pIter->pTokenMap);
+  }
+  return fts5IndexReturn(pIter->pIndex);
+}
+
 /*
 ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
 */

ext/fts5/test/fts5origintext.test

@@ -21,9 +21,13 @@ ifcapable !fts5 {
   return
 }
 
+foreach_detail_mode $testprefix {
+
 sqlite3_fts5_register_origintext db
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }
 
@@ -85,7 +89,9 @@ db func document document
 
 sqlite3_fts5_register_origintext db
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
   INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }
@@ -117,7 +123,9 @@ reset_db
 
 sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize="origintext unicode61", detail=%DETAIL%
+  );
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 
   INSERT INTO ft(rowid, x) VALUES(1, 'hello');
@@ -136,7 +144,8 @@ do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
 do_execsql_test 3.0 {
   CREATE VIRTUAL TABLE ft2 USING fts5(x, 
       tokenize="origintext unicode61", 
-      tokendata=1
+      tokendata=1,
+      detail=%DETAIL%
   );
   CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
 
@@ -169,7 +178,7 @@ sqlite3_fts5_create_function db querytoken querytoken
 
 do_execsql_test 4.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
-      x, tokenize='origintext unicode61', tokendata=1
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
   );
   INSERT INTO ft VALUES('one two three four');
 }
@@ -183,19 +192,23 @@ do_execsql_test 4.2 {
 do_execsql_test 4.3 {
   SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
 } {1 two.TWO}
-do_execsql_test 4.4 {
-  SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
-} {1 three.ThreE}
 
-do_catchsql_test 4.5 {
-  SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
-do_catchsql_test 4.6 {
-  SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
-do_catchsql_test 4.7 {
-  SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
+if {"%DETAIL%"=="full"} {
+  # Phrase queries are only supported for detail=full.
+  #
+  do_execsql_test 4.4 {
+    SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+  } {1 three.ThreE}
+  do_catchsql_test 4.5 {
+    SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.6 {
+    SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+  do_catchsql_test 4.7 {
+    SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+  } {1 SQLITE_RANGE}
+}
 
 #-------------------------------------------------------------------------
 #
@@ -210,7 +223,7 @@ fts5_aux_test_functions db
 
 do_execsql_test 5.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
-      x, tokenize='origintext unicode61', tokendata=1
+      x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
   );
   INSERT INTO ft VALUES('one ONE One oNe oNE one');
 }
@@ -239,5 +252,7 @@ do_execsql_test 5.3 {
   {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
 }
 
+}
+
 finish_test
 

ext/fts5/test/fts5origintext2.test

@@ -13,7 +13,7 @@
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5origintext
+set testprefix fts5origintext2
 
 # If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {

ext/fts5/test/fts5origintext3.test

@@ -0,0 +1,60 @@
+# 2023 November 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+  reset_db
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc insttoken {cmd iIdx iToken} { 
+    set txt [$cmd xInstToken $iIdx $iToken]
+    string map [list "\0" "."] $txt
+  }
+  sqlite3_fts5_create_function db insttoken insttoken
+  
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+    );
+  }
+  
+  do_execsql_test 1.1 {
+    INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+  }
+  
+  do_execsql_test 1.2 {
+    SELECT fts5_test_poslist(ft) FROM ft('hello');
+  } {{0.0.0 0.0.2 0.0.4}}
+
+  do_execsql_test 1.3 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hello');
+  } {hello.Hello hello.HELLO hello}
+
+}
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C When\squerying\sa\stokendata=1\sfts5\stable,\sdo\snot\suse\sa\sprefix\scursor\sfor\sthe\scase\swhere\sthe\sterm\shas\sonly\sone\svariant.
-D 2023-11-16T21:11:56.608
+C Fix\stokendata=1\sand\sxInstToken()\sAPIs\sfor\sdetail=none\sand\sdetail=column\stables.
+D 2023-11-22T19:02:54.078
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -88,13 +88,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c
-F ext/fts5/fts5Int.h 88ab1ee1eefa6f98e4c7fd3c96c99ef76ed2819cc3058736c87bb01e4a301628
+F ext/fts5/fts5Int.h d330c2e20051c300b26325b8ba29aa89e99d301c80e2f51092e5bb70346a17cd
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c 4b50ed0c724cb160f086e20e964ed2d57b99d0d3c1cb1b029901c0300b11bd9f
+F ext/fts5/fts5_expr.c 0d846134eafeeb1f0724b9c8cc02a2ef9c4082519aa3923173deadd5155910b1
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 70fa4a6d8a062ca4b63a62d0721d72ce2f6336413c6e8b0703881c708797d24d
+F ext/fts5/fts5_index.c 7b87808d788238eff4a0a68728e6ed49817e71bbfb328a18050d7d8e92a5d66a
 F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
@@ -188,8 +188,9 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
-F ext/fts5/test/fts5origintext.test 908a1fb6b1106e4b6ed0f9cf683c2ad7f986cce1aea1e0a13b3309c6f568932b
-F ext/fts5/test/fts5origintext2.test a654c77f1548ccd8eab7f6d07230655c0070cdf32dcd4740ccdf496f77d5282c
+F ext/fts5/test/fts5origintext.test 7caef7634889bab8b44d145141c0d9325299398fb89b116bccd6262fde5659db
+F ext/fts5/test/fts5origintext2.test 26482f4af1f2785cb01d06af9aae202289b6e8cf7b708d18aea305b459c2f302
+F ext/fts5/test/fts5origintext3.test 87a212b8235794348c56cb70f21e122d182a5af688c56057b90b7c151d0aa347
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2144,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07
-R d7c277a055a404d272fdcb5090bf371a
+P d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47
+R 161c23f360cac1706a2c5f6b11155312
 U dan
-Z 0e1bf556ad9eba9db356685a09c7ab31
+Z 281a4d74e3cce55af028079774718a8b
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47
+37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86