The fts3_tokenizer() function returns NULL if the

SQLITE_DBCONFIG_ENABLE_FTS_TOKENIZER setting is disabled, which is is
by default.

FossilOrigin-Name: f5732f4caf7a37a6445c61ae0d0ac14cc9deb897376e73aa36a1ead025b92c69

ext/fts3/README.tokenizers

@@ -52,8 +52,10 @@
 
   SECURITY: If the fts3 extension is used in an environment where potentially
     malicious users may execute arbitrary SQL (i.e. gears), they should be
-    prevented from invoking the fts3_tokenizer() function, possibly using the
-    authorisation callback.
+    prevented from invoking the fts3_tokenizer() function.  The
+    fts3_tokenizer() function is disabled by default. It is only enabled
+    by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in
+    security sensitive environments.
 
   See "Sample code" below for an example of calling the fts3_tokenizer()
   function from C code.

ext/fts3/fts3_tokenizer.c

@@ -106,8 +106,10 @@ static void fts3TokenizerFunc(
       return;
     }
   }
+  if( fts3TokenizerEnabled(context) ){
     sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
   }
+}
 
 int sqlite3Fts3IsIdChar(char c){
   static const char isFtsIdChar[] = {

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sminor\scomment\stypo.\s\sNo\scode\schanges.
-D 2019-03-01T21:12:40.335
+C The\sfts3_tokenizer()\sfunction\sreturns\sNULL\sif\sthe\nSQLITE_DBCONFIG_ENABLE_FTS_TOKENIZER\ssetting\sis\sdisabled,\swhich\sis\sis\nby\sdefault.
+D 2019-03-01T21:33:29.039
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 1ad7263f38329c0ecea543c80f30af839ee714ea77fc391bf1a3fbb919a5b6b5
@@ -78,7 +78,7 @@ F ext/fts2/fts2_tokenizer1.c 07e223eecb483d448313b5f1553a4f299a7fb7a1
 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
-F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
+F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
 F ext/fts3/fts3.c 5da1329ccf66b6d597dfb16b1f81aa204133c1ec96117d82a59c20126f483b17
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
@@ -93,7 +93,7 @@ F ext/fts3/fts3_snippet.c 0d8362efa59637dc7c09dc88899eb072aa409fe1e0d0fdeda55ec1
 F ext/fts3/fts3_term.c 12f7b2318f1254e6cc46dd306e5f2ac5b00b06d6761f5cae09fee5e1817cc32a
 F ext/fts3/fts3_test.c b6e9f3fd7155cb388c6bc203fb24817a721fb61d9ce28810c73fcfda8c16fda6
 F ext/fts3/fts3_tokenize_vtab.c 969c132816b6f46ee2c7efafd2547a9bfd50b0aac3f8cef3f2dca2cbd90639c7
-F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
+F ext/fts3/fts3_tokenizer.c ee670e9e0f0dc67fb78d235b2059397e4bf6a3ad8819885c2be6db08b3d35cde
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
@@ -517,7 +517,7 @@ F src/resolve.c 09419ad5c432190b69be7c0c326e03abb548a97c2c50675b81b459e1b382d1d2
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49
 F src/shell.c.in 249c0bf34f7ce272cb17162c297c45ab674a52a5d85193a86191f131196de47f
-F src/sqlite.h.in 8859e0b45b48d4186fbc466885e508f8272420a349099acdebcdb8d410d54824
+F src/sqlite.h.in f19f7b7646ccd331511b123e2e23d4dc3f3d02f74e1c04d2bb560ea50a323e4c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
 F src/sqliteInt.h f253c4ec15e577a293a462e5049f8ea1d0c7a31819b3a88acdd24698df8f4d0b
@@ -908,7 +908,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 266989148fec6d9f1bb6c5382f7aa3dcea0e9cd444576e28dd2b9287ac7dd220
-F test/fts3atoken.test 4b4c16fdcfc972f2cdbba212375a060a86ccf5f1
+F test/fts3atoken.test b7a50a58177af017ecda446e66e84d48e21e850e39e8750f1aedad0fd891450e
 F test/fts3auto.test 19097050a3ca7ab7a43b2be967cb3dfd8ddf841dfdc4eac88deb172ad2f209f2
 F test/fts3aux1.test 7a170e172afdbceb67f5baa05941fd4fbf56af42f61daa3d140f4b4bf4cb68f6
 F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f
@@ -1805,7 +1805,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9b2879629c34fc0a8e99d94648903eb93aabbc7a3682c80cb7382f9a9ca5ffb7
-R a99ec8575458a788b44781da2613b38d
+P 9a0a93c89d3fdd0f9000a9226388e2e53f299165e043913f40b83bf597bfea04
+R 6bb845ba94b22e31200ba286777e2ff0
 U drh
-Z 40bc36502bb14ccf3651e57aebfa3d90
+Z f12ee49595ceb9a1e0b9ecd89bae7dff

manifest.uuid

@@ -1 +1 @@
-9a0a93c89d3fdd0f9000a9226388e2e53f299165e043913f40b83bf597bfea04
+f5732f4caf7a37a6445c61ae0d0ac14cc9deb897376e73aa36a1ead025b92c69

src/sqlite.h.in

@@ -2086,8 +2086,8 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt>
-** <dd> ^This option is used to enable or disable the two-argument
-** version of the [fts3_tokenizer()] function which is part of the
+** <dd> ^This option is used to enable or disable the
+** [fts3_tokenizer()] function which is part of the
 ** [FTS3] full-text search engine extension.
 ** There should be two additional arguments.
 ** The first argument is an integer which is 0 to disable fts3_tokenizer() or

test/fts3atoken.test

@@ -107,6 +107,7 @@ do_test fts3atoken-2.1 {
 # simple input string via the built-in test function. This is as much
 # to test the test function as the tokenizer implementations.
 #
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
 do_test fts3atoken-3.1 {
   execsql {
     SELECT fts3_tokenizer_test('simple', 'I don''t see how');