Disable the two-argument form of the fts3_tokenizer() SQL function unless the library is built with -DSQLITE_ENABLE_FTS3_TOKENIZER.

FossilOrigin-Name: e0eb217aca7e4aadf9c44ed20822b78139f7c83c

ext/fts3/fts3_tokenizer.c

@@ -67,6 +67,7 @@ static void scalarFunc(
   nName = sqlite3_value_bytes(argv[0])+1;
 
   if( argc==2 ){
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
     void *pOld;
     int n = sqlite3_value_bytes(argv[1]);
     if( zName==0 || n!=sizeof(pPtr) ){
@@ -79,7 +80,14 @@ static void scalarFunc(
       sqlite3_result_error(context, "out of memory", -1);
       return;
     }
-  }else{
+#else
+    sqlite3_result_error(context, "fts3tokenize: " 
+        "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1
+    );
+    return;
+#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
+  }else
+  {
     if( zName ){
       pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
     }
@@ -420,11 +428,13 @@ static void intTestFunc(
   assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
 
   /* Test the storage function */
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
   rc = registerTokenizer(db, "nosuchtokenizer", p1);
   assert( rc==SQLITE_OK );
   rc = queryTokenizer(db, "nosuchtokenizer", &p2);
   assert( rc==SQLITE_OK );
   assert( p2==p1 );
+#endif
 
   sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 }

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\shelp\sthe\stokenizer\srun\sabout\s33%\sfaster.
+C Disable\sthe\stwo-argument\sform\sof\sthe\sfts3_tokenizer()\sSQL\sfunction\sunless\sthe\slibrary\sis\sbuilt\swith\s-DSQLITE_ENABLE_FTS3_TOKENIZER.
-D 2016-02-08T19:36:46.659
+D 2016-02-08T19:40:17.991
 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc fcf377286d910b47e072da1ac7945976337c0925
@@ -83,7 +83,7 @@ F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e
 F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038
 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
-F ext/fts3/fts3_tokenizer.c 50e7a69a549ac5882cc1971ee43f66aaabc11395
+F ext/fts3/fts3_tokenizer.c 0d3a77bbdc6f394d302f24980fa80667371c6e68
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145
@@ -372,7 +372,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803
 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07
 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f
-F src/test_config.c 0dee90328e3dedf8ba002ee94b6a7e7ea7726fe4
+F src/test_config.c 7985332c806d1cece793475c75a6abcccde9d331
 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c a61f54247fdb843761d709879c3bcd1989b2050c
@@ -697,7 +697,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2
-F test/fts3atoken.test e3a126365131a6db52efc20a9a6053cd44e5f289
+F test/fts3atoken.test 76262be798f23a390717d14266f0df551e52a7ee
 F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9
 F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221
 F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba
@@ -744,7 +744,7 @@ F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01
 F test/fts4growth.test 60d6bb3f78e25b34f533797dd9f2f9402310a13a
 F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269
 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d
-F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
+F test/fts4langid.test 8bd8759e0d4b04d71771544b861193a6841fee84
 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
@@ -1427,8 +1427,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 852a529a8b112049f67a3126f677c06ae4a22d73 04f7da77c13925c1f1e287f4579bb85518297d81
+P a050e6f096d40aa5b6275797b96e62c228044f5a
-R 64edf86f87d772039cf70265114474cb
+R b09b6ba902e95153c88d724675282802
-T +closed 04f7da77c13925c1f1e287f4579bb85518297d81
+U dan
-U drh
+Z a5b8919de2cf0c2997cac75a4e3c3a2c
-Z feb6daf33d3b967be60d8c209f5e5987

manifest.uuid

@@ -1 +1 @@
-a050e6f096d40aa5b6275797b96e62c228044f5a
+e0eb217aca7e4aadf9c44ed20822b78139f7c83c

src/test_config.c

@@ -370,6 +370,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
+  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_FTS5
   Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
 #else

test/fts3atoken.test

@@ -24,7 +24,7 @@ ifcapable !fts3 {
   return
 }
 
-set ::testprefix fts3token
+set ::testprefix fts3atoken
 
 proc escape_string {str} {
   set out ""
@@ -40,7 +40,7 @@ proc escape_string {str} {
 }
 
 #--------------------------------------------------------------------------
-# Test cases fts3token-1.* are the warm-body test for the SQL scalar
+# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar
 # function fts3_tokenizer(). The procedure is as follows:
 #
 #   1: Verify that there is no such fts3 tokenizer as 'blah'.
@@ -56,62 +56,68 @@ proc escape_string {str} {
 #
 #   5: Test that the table created to use tokenizer 'blah' is usable.
 #
-do_test fts3token-1.1 {
+ifcapable fts3_tokenizer {
-  catchsql {
+  do_test fts3atoken-1.1 {
-    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+    catchsql {
-  }
+      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
-} {1 {unknown tokenizer: blah}}
+    }
-do_test fts3token-1.2 {
+  } {1 {unknown tokenizer: blah}}
-  execsql {
+  do_test fts3atoken-1.2 {
+    execsql {
+      SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
+    }
+  } {0}
+  do_test fts3atoken-1.3 {
+    execsql {
+      SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
+    }
+  } {1}
+  do_test fts3atoken-1.4 {
+    catchsql {
+      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+    }
+  } {0 {}}
+  do_test fts3atoken-1.5 {
+    execsql {
+      INSERT INTO t1(content) VALUES('There was movement at the station');
+      INSERT INTO t1(content) VALUES('For the word has passed around');
+      INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
+      SELECT content FROM t1 WHERE content MATCH 'movement'
+    }
+  } {{There was movement at the station}}
+} else {
+  do_catchsql_test 1.6 {
     SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
-  }
+  } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}}
-} {0}
+}
-do_test fts3token-1.3 {
-  execsql {
-    SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
-  }
-} {1}
-do_test fts3token-1.4 {
-  catchsql {
-    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
-  }
-} {0 {}}
-do_test fts3token-1.5 {
-  execsql {
-    INSERT INTO t1(content) VALUES('There was movement at the station');
-    INSERT INTO t1(content) VALUES('For the word has passed around');
-    INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
-    SELECT content FROM t1 WHERE content MATCH 'movement'
-  }
-} {{There was movement at the station}}
 
 #--------------------------------------------------------------------------
-# Test cases fts3token-2.* test error cases in the scalar function based
+# Test cases fts3atoken-2.* test error cases in the scalar function based
 # API for getting and setting tokenizers.
 #
-do_test fts3token-2.1 {
+do_test fts3atoken-2.1 {
   catchsql {
     SELECT fts3_tokenizer('nosuchtokenizer');
   }
 } {1 {unknown tokenizer: nosuchtokenizer}}
 
 #--------------------------------------------------------------------------
-# Test cases fts3token-3.* test the three built-in tokenizers with a
+# Test cases fts3atoken-3.* test the three built-in tokenizers with a
 # simple input string via the built-in test function. This is as much
 # to test the test function as the tokenizer implementations.
 #
-do_test fts3token-3.1 {
+do_test fts3atoken-3.1 {
   execsql {
     SELECT fts3_tokenizer_test('simple', 'I don''t see how');
   }
 } {{0 i I 1 don don 2 t t 3 see see 4 how how}}
-do_test fts3token-3.2 {
+do_test fts3atoken-3.2 {
   execsql {
     SELECT fts3_tokenizer_test('porter', 'I don''t see how');
   }
 } {{0 i I 1 don don 2 t t 3 see see 4 how how}}
 ifcapable icu {
-  do_test fts3token-3.3 {
+  do_test fts3atoken-3.3 {
     execsql {
       SELECT fts3_tokenizer_test('icu', 'I don''t see how');
     }
@@ -119,7 +125,7 @@ ifcapable icu {
 }
 
 #--------------------------------------------------------------------------
-# Test cases fts3token-4.* test the ICU tokenizer. In practice, this
+# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this
 # tokenizer only has two modes - "thai" and "everybody else". Some other
 # Asian languages (Lao, Khmer etc.) require the same special treatment as 
 # Thai, but ICU doesn't support them yet.
@@ -133,8 +139,8 @@ ifcapable icu {
     } $output
   }
   
-  do_icu_test fts3token-4.1 en_US  {}   {}
+  do_icu_test fts3atoken-4.1 en_US  {}   {}
-  do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \
+  do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \
     0 test Test 1 cases cases 2 fts3 fts3
   ]
 
@@ -147,12 +153,12 @@ ifcapable icu {
   append output "1 \u0e19\u0e30 \u0e19\u0e30 "
   append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
 
-  do_icu_test fts3token-4.3 th_TH  $input $output
+  do_icu_test fts3atoken-4.3 th_TH  $input $output
-  do_icu_test fts3token-4.4 en_US  $input $output
+  do_icu_test fts3atoken-4.4 en_US  $input $output
 
   # ICU handles an unknown locale by falling back to the default.
   # So this is not an error.
-  do_icu_test fts3token-4.5 MiddleOfTheOcean  $input $output
+  do_icu_test fts3atoken-4.5 MiddleOfTheOcean  $input $output
 
   set    longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
   append longtoken "AReallocInTheIcuTokenizerCode"
@@ -164,9 +170,9 @@ ifcapable icu {
   append output "2 then then "
   append output "3 [string tolower $longtoken] $longtoken"
 
-  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
+  do_icu_test fts3atoken-4.6 MiddleOfTheOcean  $input $output
-  do_icu_test fts3token-4.7 th_TH  $input $output
+  do_icu_test fts3atoken-4.7 th_TH  $input $output
-  do_icu_test fts3token-4.8 en_US  $input $output
+  do_icu_test fts3atoken-4.8 en_US  $input $output
 
   do_execsql_test 5.1 {
     CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
@@ -186,7 +192,7 @@ ifcapable icu {
   } {}
 }
 
-do_test fts3token-internal {
+do_test fts3atoken-internal {
   execsql { SELECT fts3_tokenizer_internal_test() }
 } {ok}
 
@@ -206,12 +212,14 @@ do_catchsql_test 6.1.3 {
 do_catchsql_test 6.2.1 {
   SELECT fts3_tokenizer(NULL);
 } {1 {unknown tokenizer: }}
-do_catchsql_test 6.2.2 {
+ifcapable fts3_tokenizer {
-  SELECT fts3_tokenizer(NULL, X'1234567812345678');
+  do_catchsql_test 6.2.2 {
-} {1 {argument type mismatch}}
+    SELECT fts3_tokenizer(NULL, X'1234567812345678');
-do_catchsql_test 6.2.3 {
+  } {1 {argument type mismatch}}
-  SELECT fts3_tokenizer(NULL, X'12345678');
+  do_catchsql_test 6.2.3 {
-} {1 {argument type mismatch}}
+    SELECT fts3_tokenizer(NULL, X'12345678');
+  } {1 {argument type mismatch}}
+}
 
 
 finish_test

test/fts4langid.test

@@ -358,29 +358,31 @@ proc build_multilingual_db_2 {db} {
   }
 }
 
-do_test 4.1.0 {
+ifcapable fts3_tokenizer {
-  reset_db
+  do_test 4.1.0 {
-  set ptr [fts3_test_tokenizer]
+    reset_db
-  execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
+    set ptr [fts3_test_tokenizer]
-  build_multilingual_db_2 db
+    execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
-} {}
+    build_multilingual_db_2 db
-do_execsql_test 4.1.1 {
+  } {}
-  SELECT docid FROM t4 WHERE t4 MATCH 'quick';
+  do_execsql_test 4.1.1 {
-} {0}
+    SELECT docid FROM t4 WHERE t4 MATCH 'quick';
-do_execsql_test 4.1.2 {
+  } {0}
-  SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
+  do_execsql_test 4.1.2 {
-} {}
+    SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
-do_execsql_test 4.1.3 {
+  } {}
-  SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
+  do_execsql_test 4.1.3 {
-} {1}
+    SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
-for {set i 0} {$i < 50} {incr i} {
+  } {1}
-  do_execsql_test 4.1.4.$i {
+  for {set i 0} {$i < 50} {incr i} {
-    SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+    do_execsql_test 4.1.4.$i {
-  } [expr 0==($i%2)]
+      SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+    } [expr 0==($i%2)]
+  }
+  do_catchsql_test 4.1.5 {
+    INSERT INTO t4(content, lid) VALUES('hello world', 101)
+  } {1 {SQL logic error or missing database}}
 }
-do_catchsql_test 4.1.5 {
-  INSERT INTO t4(content, lid) VALUES('hello world', 101)
-} {1 {SQL logic error or missing database}}
 
 #-------------------------------------------------------------------------
 # Test cases 5.*