Add the sqlite3_value_frombind() interface. Use that interface to

improve fts3_tokenizer().

FossilOrigin-Name: b3f2c3205a28dc21ea7080e5e1ba246ce9c9b90c1309262ca11d8e40943ed677

ext/fts3/fts3_tokenizer.c

@@ -79,7 +79,7 @@ static void fts3TokenizerFunc(
   nName = sqlite3_value_bytes(argv[0])+1;
 
   if( argc==2 ){
-    if( fts3TokenizerEnabled(context) ){
+    if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){
       void *pOld;
       int n = sqlite3_value_bytes(argv[1]);
       if( zName==0 || n!=sizeof(pPtr) ){
@@ -106,7 +106,7 @@ static void fts3TokenizerFunc(
       return;
     }
   }
-  if( fts3TokenizerEnabled(context) ){
+  if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
   }
 }

manifest

@@ -1,5 +1,5 @@
-C Improvements\sto\sthe\ssqlite3ExprImpliesNonNullRow()\stheorem\sprover.
+C Add\sthe\ssqlite3_value_frombind()\sinterface.\s\sUse\sthat\sinterface\sto\nimprove\sfts3_tokenizer().
-D 2019-04-01T19:42:42.832
+D 2019-04-02T00:28:54.786
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts3/fts3_snippet.c 0d8362efa59637dc7c09dc88899eb072aa409fe1e0d0fdeda55ec1
 F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1
 F ext/fts3/fts3_test.c b6e9f3fd7155cb388c6bc203fb24817a721fb61d9ce28810c73fcfda8c16fda6
 F ext/fts3/fts3_tokenize_vtab.c 969c132816b6f46ee2c7efafd2547a9bfd50b0aac3f8cef3f2dca2cbd90639c7
-F ext/fts3/fts3_tokenizer.c ee670e9e0f0dc67fb78d235b2059397e4bf6a3ad8819885c2be6db08b3d35cde
+F ext/fts3/fts3_tokenizer.c 24a51bdaf5abfb83f81cad61aeaa40e9db18dd419d31607c85ac520de8bded2a
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
@@ -519,7 +519,7 @@ F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49
 F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c186796810
-F src/sqlite.h.in f765fce74038607388d3a96cd7fad892f363bdcde24911565edf610ecf69534c
+F src/sqlite.h.in e8a8dd45ae6849b3e40b243eca41181a740b4f4ac67d5b65fcea519d454db785
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
 F src/sqliteInt.h 59b625f88bf96cad4cd1a6a6ded615c161f518fb56da3e49f2ff70429e49095f
@@ -547,7 +547,7 @@ F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
 F src/test_fs.c c10706d63e64f1b1d0274e42c5443886c15ee9d6157b3bd330786aeb4057b151
-F src/test_func.c d12d805953bcb3bb19f71d29cdc93383b7b7a3369504d2b7e398a1bd77376294
+F src/test_func.c 181f992e5495644434c4f0e3cc72362a78c295eb2cf3ff4d02498b8bde7aa276
 F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d
 F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664
 F src/test_intarray.c 39b4181662a0f33a427748d87218e7578d913e683dc27eab7098bb41617cac71
@@ -589,13 +589,13 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4
 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5
 F src/util.c 82a2e3f691a3b654be872e305dab1f455e565dedf5e6a90c818c1ab307c00432
 F src/vacuum.c 532553ee1d47f69c1198858107b1f3af3d16af25f2ab4ce3b1263d9e333f26e2
-F src/vdbe.c eeaf0af94b48379b01a25566013138a68e286c82a2da68b2ab30974442518470
+F src/vdbe.c 711ef421b3bb3db3b2476067b2dc3c71ef5844d9b1a723026578f89f6da621e8
 F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237
-F src/vdbeInt.h a76d5eed62c76bcd8de7afd3147fac1bc40c5a870582664bcd7d071ef437c37f
+F src/vdbeInt.h 5ea6f6548a14b20a9210ca055a989478a37d16f84740cc123834f13e4527bce7
-F src/vdbeapi.c 7a052df80d7e2e55382076174633e888f21fe200feb5d49b1b441b4c38ab851a
+F src/vdbeapi.c a6e462bd7853e272cf614d6fbda7f3f20c89a2d255805855b32895a5983ddcec
 F src/vdbeaux.c 7aa412e7e56eb53649d87766f3064994a31991ee2b2716bf8c3129fa15cc7653
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
-F src/vdbemem.c 3e89e087df928eba80e520824078dc563c579a0848b1557ac36845ec14392923
+F src/vdbemem.c 6d67403debf6eb2b0c8d956ea09bb0ad79b4b45af2a7e2b2736894e703ae3286
 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
 F src/vtab.c 2462b7d6fd72b0b916477f5ef210ee49ab58cec195483ebdac0c8c5e3ec42cab
@@ -911,7 +911,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 266989148fec6d9f1bb6c5382f7aa3dcea0e9cd444576e28dd2b9287ac7dd220
-F test/fts3atoken.test b7a50a58177af017ecda446e66e84d48e21e850e39e8750f1aedad0fd891450e
+F test/fts3atoken.test bef8a163490098a6b8a6ec5f5407269a3a15b9902c0fcf5e962825a81675b3a0
 F test/fts3auto.test 19097050a3ca7ab7a43b2be967cb3dfd8ddf841dfdc4eac88deb172ad2f209f2
 F test/fts3aux1.test 7a170e172afdbceb67f5baa05941fd4fbf56af42f61daa3d140f4b4bf4cb68f6
 F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f
@@ -975,7 +975,7 @@ F test/fts4opt.test 0fd0cc84000743ff2a883b9b84b4a5be07249f0ba790c8848a757164cdd4
 F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f757380429
 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
-F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7
+F test/func.test 150270b6e2e0281697c116e5ca0e46b41ace8d34b1c92461d88fdd9968c9b03c
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c
 F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f
@@ -1815,7 +1815,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b36813d6467c82159bd3bb69d34ac28fc161a13052ca67d7cf9ad75e2aaea9d5
+P 3fde627616030ca0de87169467e2e453fdc91154942e3a3a92a221df5923b2d2 1dc7993bb6957587a6c63142e97b8e4137c3d81bbfd724f86048e0894bcb429b
-R 53cefdd43a017d7f72ed1400359072e1
+R c147f0404a98a8ecfae0b6c4ab1ffd42
 U drh
-Z 294bced1998ead69a391511a2008e481
+Z 218a8691f3a518f79e729d9161496492

manifest.uuid

@@ -1 +1 @@
-3fde627616030ca0de87169467e2e453fdc91154942e3a3a92a221df5923b2d2
+b3f2c3205a28dc21ea7080e5e1ba246ce9c9b90c1309262ca11d8e40943ed677

src/sqlite.h.in

@@ -4979,6 +4979,8 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** <tr><td><b>sqlite3_value_nochange&nbsp;&nbsp;</b>
 ** <td>&rarr;&nbsp;&nbsp;<td>True if the column is unchanged in an UPDATE
 ** against a virtual table.
+** <tr><td><b>sqlite3_value_frombind&nbsp;&nbsp;</b>
+** <td>&rarr;&nbsp;&nbsp;<td>True if value originated a bound parameter
 ** </table></blockquote>
 **
 ** <b>Details:</b>
@@ -5040,6 +5042,11 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** than within an [xUpdate] method call for an UPDATE statement, then
 ** the return value is arbitrary and meaningless.
 **
+** ^The sqlite3_value_frombind(X) interface returns non-zero if the
+** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()]
+** interfaces.  ^If X comes from an SQL literal value, or a table column,
+** and expression, then sqlite3_value_frombind(X) returns zero.
+**
 ** Please pay particular attention to the fact that the pointer returned
 ** from [sqlite3_value_blob()], [sqlite3_value_text()], or
 ** [sqlite3_value_text16()] can be invalidated by a subsequent call to
@@ -5085,6 +5092,7 @@ int sqlite3_value_bytes16(sqlite3_value*);
 int sqlite3_value_type(sqlite3_value*);
 int sqlite3_value_numeric_type(sqlite3_value*);
 int sqlite3_value_nochange(sqlite3_value*);
+int sqlite3_value_frombind(sqlite3_value*);
 
 /*
 ** CAPI3REF: Finding The Subtype Of SQL Values

src/test_func.c

@@ -630,6 +630,24 @@ static void test_getsubtype(
   sqlite3_result_int(context, (int)sqlite3_value_subtype(argv[0]));
 }
 
+/*         test_frombind(A,B,C,...)
+**
+** Return an integer bitmask that has a bit set for every argument
+** (up to the first 63 arguments) that originates from a bind a parameter.
+*/
+static void test_frombind(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3_uint64 m = 0;
+  int i;
+  for(i=0; i<argc && i<63; i++){
+    if( sqlite3_value_frombind(argv[i]) ) m |= ((sqlite3_uint64)1)<<i;
+  }
+  sqlite3_result_int64(context, (sqlite3_int64)m);
+}
+
 /*         test_setsubtype(V, T)
 **
 ** Return the value V with its subtype changed to T
@@ -675,6 +693,7 @@ static int registerTestFunctions(
     { "test_zeroblob",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob},
     { "test_getsubtype",       1, SQLITE_UTF8, test_getsubtype},
     { "test_setsubtype",       2, SQLITE_UTF8, test_setsubtype},
+    { "test_frombind",        -1, SQLITE_UTF8, test_frombind},
   };
   int i;
 

src/vdbe.c

@@ -1256,7 +1256,10 @@ case OP_Variable: {            /* out2 */
     goto too_big;
   }
   pOut = &aMem[pOp->p2];
-  sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static);
+  if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut);
+  memcpy(pOut, pVar, MEMCELLSIZE);
+  pOut->flags &= ~(MEM_Dyn|MEM_Ephem);
+  pOut->flags |= MEM_Static|MEM_FromBind;
   UPDATE_MAX_BLOBSIZE(pOut);
   break;
 }

src/vdbeInt.h

@@ -246,7 +246,7 @@ struct sqlite3_value {
 #define MEM_Real      0x0008   /* Value is a real number */
 #define MEM_Blob      0x0010   /* Value is a BLOB */
 #define MEM_AffMask   0x001f   /* Mask of affinity bits */
-/* Available          0x0020   */
+#define MEM_FromBind  0x0020   /* Value originates from sqlite3_bind() */
 /* Available          0x0040   */
 #define MEM_Undefined 0x0080   /* Value is undefined */
 #define MEM_Cleared   0x0100   /* NULL set by OP_Null, not from data */

src/vdbeapi.c

@@ -275,6 +275,11 @@ int sqlite3_value_nochange(sqlite3_value *pVal){
   return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero);
 }
 
+/* Return true if a parameter value originated from an sqlite3_bind() */
+int sqlite3_value_frombind(sqlite3_value *pVal){
+  return (pVal->flags&MEM_FromBind)!=0;
+}
+
 /* Make a copy of an sqlite3_value object
 */
 sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){

src/vdbemem.c

@@ -57,7 +57,7 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){
               ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 );
 
       /* No other bits set */
-      assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype
+      assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind
                            |MEM_Dyn|MEM_Ephem|MEM_Static))==0 );
     }else{
       /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn,

test/fts3atoken.test

@@ -86,11 +86,48 @@ do_test fts3atoken-1.5 {
   }
 } {{There was movement at the station}}
 
+unset -nocomplain simple blah2name simplename
+set simplename "simple"
+set blah2name "blah2"
+set simple [db one {SELECT fts3_tokenizer('simple')}]
 sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
 do_catchsql_test 1.6 {
   SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
 } {1 {fts3tokenize disabled}}
+do_test fts3atoken-1.7 {
+  execsql {
+    SELECT fts3_tokenizer('blah2', $simple) IS NULL;
+  }
+} {1}
 
+# With ENABLE_FTS3_TOKENIZER off, the fts3_tokenzer(1) function
+# returns NULL unless the first parameter is a bound parameter.
+# If the first parameter is a bound parameter, then fts3_tokenizer(1)
+# returns the actual pointer value as a BLOB.
+#
+do_test fts3atoken-1.8 {
+  execsql {
+    SELECT fts3_tokenizer($blah2name) == fts3_tokenizer($simplename),
+           typeof(fts3_tokenizer($blah2name)),
+           typeof(fts3_tokenizer('blah2')),
+           typeof(fts3_tokenizer($simplename)),
+           typeof(fts3_tokenizer('simple'));
+  }
+} {1 blob null blob null}
+
+# With ENABLE_FTS3_TOKENIZER on, fts3_tokenizer() always returns
+# the BLOB pointer, regardless the parameter
+#
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+do_test fts3atoken-1.9 {
+  execsql {
+    SELECT fts3_tokenizer('blah2') == fts3_tokenizer('simple'),
+           typeof(fts3_tokenizer($blah2name)),
+           typeof(fts3_tokenizer('blah2')),
+           typeof(fts3_tokenizer($simplename)),
+           typeof(fts3_tokenizer('simple'));
+  }
+} {1 blob blob blob blob}
 
 #--------------------------------------------------------------------------
 # Test cases fts3atoken-2.* test error cases in the scalar function based

test/func.test

@@ -1391,4 +1391,32 @@ for {set i 65536} {$i<=0x10ffff} {incr i 139} {
 do_execsql_test func-31.1 { 
   SELECT char(), length(char()), typeof(char()) 
 } {{} 0 text}
+
+# sqlite3_value_frombind()
+#
+do_execsql_test func-32.100 {
+  SELECT test_frombind(1,2,3,4);
+} {0}
+do_execsql_test func-32.110 {
+  SELECT test_frombind(1,2,?,4);
+} {4}
+do_execsql_test func-32.120 {
+  SELECT test_frombind(1,(?),4,?+7);
+} {2}
+do_execsql_test func-32.130 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b,c,e,f);
+  INSERT INTO t1 VALUES(1,2.5,'xyz',x'e0c1b2a3',null);
+  SELECT test_frombind(a,b,c,e,f,$xyz) FROM t1;
+} {32}
+do_execsql_test func-32.140 {
+  SELECT test_frombind(a,b,c,e,f,$xyz+f) FROM t1;
+} {0}
+do_execsql_test func-32.150 {
+  SELECT test_frombind(x.a,y.b,x.c,:123,y.e,x.f,$xyz+y.f) FROM t1 x, t1 y;
+} {8}
+
+
+
+
 finish_test