Fix a problem with the xInstCount() API and "ORDER BY rank" queries.

FossilOrigin-Name: 317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc

ext/fts5/fts5Int.h

@@ -787,7 +787,7 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 
 int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
-int sqlite3Fts5ExprInstToken(Fts5Expr*, int, int, int, int, const char**, int*);
+int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
 void sqlite3Fts5ExprClearTokens(Fts5Expr*);
 
 /*******************************************

ext/fts5/fts5_expr.c

@@ -3199,6 +3199,7 @@ int sqlite3Fts5ExprQueryToken(
 */
 int sqlite3Fts5ExprInstToken(
   Fts5Expr *pExpr, 
+  i64 iRowid,
   int iPhrase, 
   int iCol, 
   int iOff, 
@@ -3208,7 +3209,6 @@ int sqlite3Fts5ExprInstToken(
 ){
   Fts5ExprPhrase *pPhrase = 0;
   Fts5IndexIter *pIter = 0;
-  i64 iRowid = pExpr->pRoot->iRowid;
 
   if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
     return SQLITE_RANGE;

ext/fts5/fts5_main.c

@@ -2364,9 +2364,10 @@ static int fts5ApiInstToken(
       int iPhrase = pCsr->aInst[iIdx*3];
       int iCol = pCsr->aInst[iIdx*3 + 1];
       int iOff = pCsr->aInst[iIdx*3 + 2];
+      i64 iRowid = fts5CursorRowid(pCsr);
 
       rc = sqlite3Fts5ExprInstToken(
-          pCsr->pExpr, iPhrase, iCol, iOff, iToken, ppOut, pnOut
+          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
       );
     }
   }

ext/fts5/test/fts5origintext5.test

@@ -79,15 +79,17 @@ db func document document
 
 #-------------------------------------------------------------------------
 
+expr srand(6)
+
 set NDOC  200
 set NLOOP 100
 
 sqlite3_fts5_register_origintext db
 
 proc tokens {cmd} { 
+  set ret [list]
   for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
     set txt [$cmd xInstToken $iTok 0]
-    if {$txt==""} break
     set txt [string map [list "\0" "."] $txt]
     lappend ret $txt
   }
@@ -95,6 +97,11 @@ proc tokens {cmd} {
 }
 sqlite3_fts5_create_function db tokens tokens
 
+proc rankfunc {cmd} { 
+  $cmd xRowid
+}
+sqlite3_fts5_create_function db rankfunc rankfunc
+
 proc ctrl_tokens {doc term} {
   set ret [list]
   set term [string tolower $term]
@@ -114,12 +121,13 @@ db func ctrl_tokens ctrl_tokens
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
-      x, tokenize="origintext unicode61", 
+      x, tokenize="origintext unicode61", content=, contentless_delete=1,
       tokendata=1
   );
 
   CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
   INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+  INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
 }
 do_test 1.1 {
   for {set ii 0} {$ii < $NDOC} {incr ii} {
@@ -136,9 +144,12 @@ proc do_all_vocab_test {tn} {
     set answer [execsql {
       SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%'
     }]
-    do_execsql_test $tn.$::v {
+    do_execsql_test $tn.$::v.1 {
       SELECT rowid, tokens(ft) FROM ft($::v)
     } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
   }
 }
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sbug\sin\sxInstToken()\scausing\sthe\swrong\stoken\sto\sbe\sreturned.
-D 2023-12-04T17:05:37.721
+C Fix\sa\sproblem\swith\sthe\sxInstCount()\sAPI\sand\s"ORDER\sBY\srank"\squeries.
+D 2023-12-04T17:45:33.714
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -90,14 +90,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff90acaa97f8e865b66d1177d1b56b8c110fd5548ab5863bab43f055a1d745fe
-F ext/fts5/fts5Int.h 1fdbf3d16bdd481fe2ee99927919e4c3db835efae00f8efd7efb5e6a93277459
+F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796
+F ext/fts5/fts5_expr.c aec893108d90cc8eae6801283d38d4e705d8c22aba12717fcc56e1910b8e3b32
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10
-F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0
+F ext/fts5/fts5_main.c 1fbadf63a7381fd68753efdccce19683f587668489b1c5a9c7bf53d6e9b64872
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0
 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
-F ext/fts5/test/fts5origintext5.test f4377b67debb10e3731030ce51245b9843ffb31f85725615b3b8820bd5912702
+F ext/fts5/test/fts5origintext5.test f9dfb005248d764fdc52859308875f680b523a897258a2cc40100f9e2356b5a9
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10
-R 135aeb3f5184e1f0131c446ddc8eab37
+P da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f
+R 17d195e142931273edf7c88c21ff2164
 U dan
-Z 33215e49f8892bbcd31004428ffced69
+Z 7652ccb47cdd7a8e28db861627789490
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f
+317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc