diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index 64cfe07aac..26bac57648 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -67,6 +67,7 @@ static void scalarFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ @@ -79,7 +80,14 @@ static void scalarFunc( sqlite3_result_error(context, "out of memory", -1); return; } - }else{ +#else + sqlite3_result_error(context, "fts3tokenize: " + "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1 + ); + return; +#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */ + }else + { if( zName ){ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); } @@ -420,11 +428,13 @@ static void intTestFunc( assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER rc = registerTokenizer(db, "nosuchtokenizer", p1); assert( rc==SQLITE_OK ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_OK ); assert( p2==p1 ); +#endif sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } diff --git a/manifest b/manifest index 2902207223..2ad9b86f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\shelp\sthe\stokenizer\srun\sabout\s33%\sfaster. -D 2016-02-08T19:36:46.659 +C Disable\sthe\stwo-argument\sform\sof\sthe\sfts3_tokenizer()\sSQL\sfunction\sunless\sthe\slibrary\sis\sbuilt\swith\s-DSQLITE_ENABLE_FTS3_TOKENIZER. +D 2016-02-08T19:40:17.991 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc fcf377286d910b47e072da1ac7945976337c0925 @@ -83,7 +83,7 @@ F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 -F ext/fts3/fts3_tokenizer.c 50e7a69a549ac5882cc1971ee43f66aaabc11395 +F ext/fts3/fts3_tokenizer.c 0d3a77bbdc6f394d302f24980fa80667371c6e68 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 @@ -372,7 +372,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 0dee90328e3dedf8ba002ee94b6a7e7ea7726fe4 +F src/test_config.c 7985332c806d1cece793475c75a6abcccde9d331 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c a61f54247fdb843761d709879c3bcd1989b2050c @@ -697,7 +697,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2 -F test/fts3atoken.test e3a126365131a6db52efc20a9a6053cd44e5f289 +F test/fts3atoken.test 76262be798f23a390717d14266f0df551e52a7ee F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9 F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221 F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba @@ -744,7 +744,7 @@ F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 60d6bb3f78e25b34f533797dd9f2f9402310a13a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d -F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 +F test/fts4langid.test 8bd8759e0d4b04d71771544b861193a6841fee84 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 @@ -1427,8 +1427,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 852a529a8b112049f67a3126f677c06ae4a22d73 04f7da77c13925c1f1e287f4579bb85518297d81 -R 64edf86f87d772039cf70265114474cb -T +closed 04f7da77c13925c1f1e287f4579bb85518297d81 -U drh -Z feb6daf33d3b967be60d8c209f5e5987 +P a050e6f096d40aa5b6275797b96e62c228044f5a +R b09b6ba902e95153c88d724675282802 +U dan +Z a5b8919de2cf0c2997cac75a4e3c3a2c diff --git a/manifest.uuid b/manifest.uuid index dc1ddc7e40..fcaa3cc3b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a050e6f096d40aa5b6275797b96e62c228044f5a \ No newline at end of file +e0eb217aca7e4aadf9c44ed20822b78139f7c83c \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 30b421e00b..5db7117555 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -370,6 +370,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER + Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_ENABLE_FTS5 Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY); #else diff --git a/test/fts3atoken.test b/test/fts3atoken.test index 88e3d4f72b..2cdea79a98 100644 --- a/test/fts3atoken.test +++ b/test/fts3atoken.test @@ -24,7 +24,7 @@ ifcapable !fts3 { return } -set ::testprefix fts3token +set ::testprefix fts3atoken proc escape_string {str} { set out "" @@ -40,7 +40,7 @@ proc escape_string {str} { } #-------------------------------------------------------------------------- -# Test cases fts3token-1.* are the warm-body test for the SQL scalar +# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar # function fts3_tokenizer(). The procedure is as follows: # # 1: Verify that there is no such fts3 tokenizer as 'blah'. @@ -56,62 +56,68 @@ proc escape_string {str} { # # 5: Test that the table created to use tokenizer 'blah' is usable. # -do_test fts3token-1.1 { - catchsql { - CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); - } -} {1 {unknown tokenizer: blah}} -do_test fts3token-1.2 { - execsql { +ifcapable fts3_tokenizer { + do_test fts3atoken-1.1 { + catchsql { + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); + } + } {1 {unknown tokenizer: blah}} + do_test fts3atoken-1.2 { + execsql { + SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; + } + } {0} + do_test fts3atoken-1.3 { + execsql { + SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); + } + } {1} + do_test fts3atoken-1.4 { + catchsql { + CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); + } + } {0 {}} + do_test fts3atoken-1.5 { + execsql { + INSERT INTO t1(content) VALUES('There was movement at the station'); + INSERT INTO t1(content) VALUES('For the word has passed around'); + INSERT INTO t1(content) VALUES('That the colt from ol regret had got'); + SELECT content FROM t1 WHERE content MATCH 'movement' + } + } {{There was movement at the station}} +} else { + do_catchsql_test 1.6 { SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; - } -} {0} -do_test fts3token-1.3 { - execsql { - SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); - } -} {1} -do_test fts3token-1.4 { - catchsql { - CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); - } -} {0 {}} -do_test fts3token-1.5 { - execsql { - INSERT INTO t1(content) VALUES('There was movement at the station'); - INSERT INTO t1(content) VALUES('For the word has passed around'); - INSERT INTO t1(content) VALUES('That the colt from ol regret had got away'); - SELECT content FROM t1 WHERE content MATCH 'movement' - } -} {{There was movement at the station}} + } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}} +} #-------------------------------------------------------------------------- -# Test cases fts3token-2.* test error cases in the scalar function based +# Test cases fts3atoken-2.* test error cases in the scalar function based # API for getting and setting tokenizers. # -do_test fts3token-2.1 { +do_test fts3atoken-2.1 { catchsql { SELECT fts3_tokenizer('nosuchtokenizer'); } } {1 {unknown tokenizer: nosuchtokenizer}} #-------------------------------------------------------------------------- -# Test cases fts3token-3.* test the three built-in tokenizers with a +# Test cases fts3atoken-3.* test the three built-in tokenizers with a # simple input string via the built-in test function. This is as much # to test the test function as the tokenizer implementations. # -do_test fts3token-3.1 { +do_test fts3atoken-3.1 { execsql { SELECT fts3_tokenizer_test('simple', 'I don''t see how'); } } {{0 i I 1 don don 2 t t 3 see see 4 how how}} -do_test fts3token-3.2 { +do_test fts3atoken-3.2 { execsql { SELECT fts3_tokenizer_test('porter', 'I don''t see how'); } } {{0 i I 1 don don 2 t t 3 see see 4 how how}} ifcapable icu { - do_test fts3token-3.3 { + do_test fts3atoken-3.3 { execsql { SELECT fts3_tokenizer_test('icu', 'I don''t see how'); } @@ -119,7 +125,7 @@ ifcapable icu { } #-------------------------------------------------------------------------- -# Test cases fts3token-4.* test the ICU tokenizer. In practice, this +# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this # tokenizer only has two modes - "thai" and "everybody else". Some other # Asian languages (Lao, Khmer etc.) require the same special treatment as # Thai, but ICU doesn't support them yet. @@ -133,8 +139,8 @@ ifcapable icu { } $output } - do_icu_test fts3token-4.1 en_US {} {} - do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \ + do_icu_test fts3atoken-4.1 en_US {} {} + do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \ 0 test Test 1 cases cases 2 fts3 fts3 ] @@ -147,12 +153,12 @@ ifcapable icu { append output "1 \u0e19\u0e30 \u0e19\u0e30 " append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a" - do_icu_test fts3token-4.3 th_TH $input $output - do_icu_test fts3token-4.4 en_US $input $output + do_icu_test fts3atoken-4.3 th_TH $input $output + do_icu_test fts3atoken-4.4 en_US $input $output # ICU handles an unknown locale by falling back to the default. # So this is not an error. - do_icu_test fts3token-4.5 MiddleOfTheOcean $input $output + do_icu_test fts3atoken-4.5 MiddleOfTheOcean $input $output set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire" append longtoken "AReallocInTheIcuTokenizerCode" @@ -164,9 +170,9 @@ ifcapable icu { append output "2 then then " append output "3 [string tolower $longtoken] $longtoken" - do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output - do_icu_test fts3token-4.7 th_TH $input $output - do_icu_test fts3token-4.8 en_US $input $output + do_icu_test fts3atoken-4.6 MiddleOfTheOcean $input $output + do_icu_test fts3atoken-4.7 th_TH $input $output + do_icu_test fts3atoken-4.8 en_US $input $output do_execsql_test 5.1 { CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); @@ -186,7 +192,7 @@ ifcapable icu { } {} } -do_test fts3token-internal { +do_test fts3atoken-internal { execsql { SELECT fts3_tokenizer_internal_test() } } {ok} @@ -206,12 +212,14 @@ do_catchsql_test 6.1.3 { do_catchsql_test 6.2.1 { SELECT fts3_tokenizer(NULL); } {1 {unknown tokenizer: }} -do_catchsql_test 6.2.2 { - SELECT fts3_tokenizer(NULL, X'1234567812345678'); -} {1 {argument type mismatch}} -do_catchsql_test 6.2.3 { - SELECT fts3_tokenizer(NULL, X'12345678'); -} {1 {argument type mismatch}} +ifcapable fts3_tokenizer { + do_catchsql_test 6.2.2 { + SELECT fts3_tokenizer(NULL, X'1234567812345678'); + } {1 {argument type mismatch}} + do_catchsql_test 6.2.3 { + SELECT fts3_tokenizer(NULL, X'12345678'); + } {1 {argument type mismatch}} +} finish_test diff --git a/test/fts4langid.test b/test/fts4langid.test index 843e11f9ab..a3059931e7 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -358,29 +358,31 @@ proc build_multilingual_db_2 {db} { } } -do_test 4.1.0 { - reset_db - set ptr [fts3_test_tokenizer] - execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } - build_multilingual_db_2 db -} {} -do_execsql_test 4.1.1 { - SELECT docid FROM t4 WHERE t4 MATCH 'quick'; -} {0} -do_execsql_test 4.1.2 { - SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; -} {} -do_execsql_test 4.1.3 { - SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; -} {1} -for {set i 0} {$i < 50} {incr i} { - do_execsql_test 4.1.4.$i { - SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; - } [expr 0==($i%2)] +ifcapable fts3_tokenizer { + do_test 4.1.0 { + reset_db + set ptr [fts3_test_tokenizer] + execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } + build_multilingual_db_2 db + } {} + do_execsql_test 4.1.1 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick'; + } {0} + do_execsql_test 4.1.2 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; + } {} + do_execsql_test 4.1.3 { + SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; + } {1} + for {set i 0} {$i < 50} {incr i} { + do_execsql_test 4.1.4.$i { + SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; + } [expr 0==($i%2)] + } + do_catchsql_test 4.1.5 { + INSERT INTO t4(content, lid) VALUES('hello world', 101) + } {1 {SQL logic error or missing database}} } -do_catchsql_test 4.1.5 { - INSERT INTO t4(content, lid) VALUES('hello world', 101) -} {1 {SQL logic error or missing database}} #------------------------------------------------------------------------- # Test cases 5.*