mirror of
https://github.com/sqlite/sqlite.git
synced 2025-11-18 10:21:03 +03:00
Merge the unicode61 tokenizer and the shared-cache-memory database changes
into the sessions branch. FossilOrigin-Name: df817e70afc3f41e680d8f84dfa5772d5b3ae4d9
This commit is contained in:
drh
@@ -254,6 +254,8 @@ foreach {tn uri error} "
|
||||
4 {file:test.db?mode=Ro} {no such access mode: Ro}
|
||||
5 {file:test.db?mode=Rw} {no such access mode: Rw}
|
||||
6 {file:test.db?mode=Rwc} {no such access mode: Rwc}
|
||||
7 {file:test.db?mode=memory} {not an error}
|
||||
8 {file:test.db?mode=MEMORY} {no such access mode: MEMORY}
|
||||
" {
|
||||
do_test 7.$tn { open_uri_error $uri } $error
|
||||
}
|
||||
|
||||
@@ -131,4 +131,28 @@ do_faultsim_test 4.1 -prep {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
ifcapable fts3_unicode {
|
||||
do_test 5.0 {
|
||||
faultsim_delete_and_reopen
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
|
||||
}
|
||||
faultsim_save_and_close
|
||||
} {}
|
||||
|
||||
do_faultsim_test 5.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
db eval {SELECT * FROM sqlite_master}
|
||||
} -body {
|
||||
execsql { INSERT INTO ft VALUES('the quick brown fox'); }
|
||||
execsql { INSERT INTO ft VALUES(
|
||||
'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
|
||||
);
|
||||
}
|
||||
execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
|
||||
} -test {
|
||||
faultsim_test_result {0 {1 2}}
|
||||
}
|
||||
}
|
||||
|
||||
finish_test
|
||||
|
||||
227
test/fts4unicode.test
Normal file
227
test/fts4unicode.test
Normal file
@@ -0,0 +1,227 @@
|
||||
# 2012 May 25
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
# The tests in this file focus on testing the "unicode" FTS tokenizer.
|
||||
#
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
source $testdir/tester.tcl
|
||||
ifcapable !fts3_unicode { finish_test ; return }
|
||||
set ::testprefix fts4unicode
|
||||
|
||||
proc do_unicode_token_test {tn input res} {
|
||||
set input [string map {' ''} $input]
|
||||
uplevel [list do_execsql_test $tn "
|
||||
SELECT fts3_tokenizer_test('unicode61', '$input');
|
||||
" [list [list {*}$res]]]
|
||||
}
|
||||
|
||||
do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D}
|
||||
do_unicode_token_test 1.1 {<7B> <20> <20>} {0 <20> <20> 1 <20> <20> 2 <20> <20>}
|
||||
do_unicode_token_test 1.2 {x<>x x<>x x<>x} {0 x<>x x<>x 1 x<>x x<>x 2 x<>x x<>x}
|
||||
|
||||
# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s.
|
||||
do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF"
|
||||
do_unicode_token_test 1.4 "\u1E9E" "0 <20> \u1E9E"
|
||||
do_unicode_token_test 1.5 "\u1E9E" "0 \uDF \u1E9E"
|
||||
|
||||
do_unicode_token_test 1.6 "The quick brown fox" {
|
||||
0 the The 1 quick quick 2 brown brown 3 fox fox
|
||||
}
|
||||
do_unicode_token_test 1.7 "The\u00bfquick\u224ebrown\u2263fox" {
|
||||
0 the The 1 quick quick 2 brown brown 3 fox fox
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
set docs [list {
|
||||
Enhance the INSERT syntax to allow multiple rows to be inserted via the
|
||||
VALUES clause.
|
||||
} {
|
||||
Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
|
||||
} {
|
||||
Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
|
||||
} {
|
||||
Added the sqlite3_db_readonly() interface.
|
||||
} {
|
||||
Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
|
||||
ability to add new PRAGMA statements or to override built-in PRAGMAs.
|
||||
} {
|
||||
Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
|
||||
the same row that contains the maximum x value.
|
||||
} {
|
||||
Added support for the FTS4 languageid option.
|
||||
} {
|
||||
Documented support for the FTS4 content option. This feature has actually
|
||||
been in the code since version 3.7.9 but is only now considered to be
|
||||
officially supported.
|
||||
} {
|
||||
Pending statements no longer block ROLLBACK. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the ROLLBACK.
|
||||
} {
|
||||
Improvements to the handling of CSV inputs in the command-line shell
|
||||
} {
|
||||
Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
|
||||
incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
|
||||
connected by OR.
|
||||
}]
|
||||
|
||||
set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS
|
||||
set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS
|
||||
set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS
|
||||
set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS
|
||||
set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS
|
||||
set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS
|
||||
set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS
|
||||
set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS
|
||||
set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS
|
||||
foreach k [array names map] {
|
||||
lappend mappings [string toupper $k] [lindex $map($k) 0]
|
||||
lappend mappings $k [lindex $map($k) 1]
|
||||
}
|
||||
proc mapdoc {doc} {
|
||||
set doc [regsub -all {[[:space:]]+} $doc " "]
|
||||
string map $::mappings [string trim $doc]
|
||||
}
|
||||
|
||||
do_test 2.0 {
|
||||
execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
|
||||
foreach doc $docs {
|
||||
set d [mapdoc $doc]
|
||||
execsql { INSERT INTO t2 VALUES($d) }
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 2.1 {
|
||||
set q [mapdoc "row"]
|
||||
execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
|
||||
} [list [mapdoc {
|
||||
Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
|
||||
the same row that contains the maximum x value.
|
||||
}]]
|
||||
|
||||
foreach {tn query snippet} {
|
||||
2 "row" {
|
||||
...returns the value of y on the same [row] that contains
|
||||
the maximum x value.
|
||||
}
|
||||
3 "ROW" {
|
||||
...returns the value of y on the same [row] that contains
|
||||
the maximum x value.
|
||||
}
|
||||
4 "rollback" {
|
||||
...[ROLLBACK]. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the [ROLLBACK].
|
||||
}
|
||||
5 "rOllback" {
|
||||
...[ROLLBACK]. Instead, the pending statement
|
||||
will return SQLITE_ABORT upon next access after the [ROLLBACK].
|
||||
}
|
||||
6 "lang*" {
|
||||
Added support for the FTS4 [languageid] option.
|
||||
}
|
||||
} {
|
||||
do_test 2.$tn {
|
||||
set q [mapdoc $query]
|
||||
execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
|
||||
} [list [mapdoc $snippet]]
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Make sure the unicode61 tokenizer does not crash if it is passed a
|
||||
# NULL pointer.
|
||||
reset_db
|
||||
do_execsql_test 3.1 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y);
|
||||
INSERT INTO t1 VALUES(NULL, 'a b c');
|
||||
}
|
||||
|
||||
do_execsql_test 3.2 {
|
||||
SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b'
|
||||
} {{a [b] c}}
|
||||
|
||||
do_execsql_test 3.3 {
|
||||
BEGIN;
|
||||
DELETE FROM t1;
|
||||
INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 SELECT * FROM t1;
|
||||
INSERT INTO t1 VALUES('a b c', NULL);
|
||||
INSERT INTO t1 VALUES('a x c', NULL);
|
||||
COMMIT;
|
||||
}
|
||||
|
||||
do_execsql_test 3.4 {
|
||||
SELECT * FROM t1 WHERE t1 MATCH 'a b';
|
||||
} {{a b c} {}}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
|
||||
do_test 4.1 {
|
||||
set a "abc\uFFFEdef"
|
||||
set b "abc\uD800def"
|
||||
set c "\uFFFEdef"
|
||||
set d "\uD800def"
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x);
|
||||
INSERT INTO t1 VALUES($a);
|
||||
INSERT INTO t1 VALUES($b);
|
||||
INSERT INTO t1 VALUES($c);
|
||||
INSERT INTO t1 VALUES($d);
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 4.2 {
|
||||
set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
|
||||
set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
|
||||
set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
|
||||
set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
|
||||
execsql {
|
||||
INSERT INTO t1 VALUES($a);
|
||||
INSERT INTO t1 VALUES($b);
|
||||
INSERT INTO t1 VALUES($c);
|
||||
INSERT INTO t1 VALUES($d);
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 4.3 {
|
||||
set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
|
||||
set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
|
||||
set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
|
||||
set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
|
||||
execsql {
|
||||
INSERT INTO t1 VALUES($a);
|
||||
INSERT INTO t1 VALUES($b);
|
||||
INSERT INTO t1 VALUES($c);
|
||||
INSERT INTO t1 VALUES($d);
|
||||
}
|
||||
} {}
|
||||
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
@@ -188,7 +188,7 @@ test_suite "fts3" -prefix "" -description {
|
||||
fts4aa.test fts4content.test
|
||||
fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
|
||||
fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
|
||||
fts4check.test
|
||||
fts4check.test fts4unicode.test
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1056,7 +1056,96 @@ do_test shared-$av-15.2 {
|
||||
db close
|
||||
db2 close
|
||||
|
||||
}
|
||||
# Shared cache on a :memory: database. This only works for URI filenames.
|
||||
#
|
||||
do_test shared-$av-16.1 {
|
||||
sqlite3 db1 file::memory: -uri 1
|
||||
sqlite3 db2 file::memory: -uri 1
|
||||
db1 eval {
|
||||
CREATE TABLE t1(x); INSERT INTO t1 VALUES(1),(2),(3);
|
||||
}
|
||||
db2 eval {
|
||||
SELECT x FROM t1 ORDER BY x;
|
||||
}
|
||||
} {1 2 3}
|
||||
do_test shared-$av-16.2 {
|
||||
db2 eval {
|
||||
INSERT INTO t1 VALUES(99);
|
||||
DELETE FROM t1 WHERE x=2;
|
||||
}
|
||||
db1 eval {
|
||||
SELECT x FROM t1 ORDER BY x;
|
||||
}
|
||||
} {1 3 99}
|
||||
|
||||
# Verify that there is no cache sharing ordinary (non-URI) filenames are
|
||||
# used.
|
||||
#
|
||||
do_test shared-$av-16.3 {
|
||||
db1 close
|
||||
db2 close
|
||||
sqlite3 db1 :memory:
|
||||
sqlite3 db2 :memory:
|
||||
db1 eval {
|
||||
CREATE TABLE t1(x); INSERT INTO t1 VALUES(4),(5),(6);
|
||||
}
|
||||
catchsql {
|
||||
SELECT * FROM t1;
|
||||
} db2
|
||||
} {1 {no such table: t1}}
|
||||
|
||||
# Shared cache on named memory databases.
|
||||
#
|
||||
do_test shared-$av-16.4 {
|
||||
db1 close
|
||||
db2 close
|
||||
forcedelete test.db test.db-wal test.db-journal
|
||||
sqlite3 db1 file:test.db?mode=memory -uri 1
|
||||
sqlite3 db2 file:test.db?mode=memory -uri 1
|
||||
db1 eval {
|
||||
CREATE TABLE t1(x); INSERT INTO t1 VALUES(1),(2),(3);
|
||||
}
|
||||
db2 eval {
|
||||
SELECT x FROM t1 ORDER BY x;
|
||||
}
|
||||
} {1 2 3}
|
||||
do_test shared-$av-16.5 {
|
||||
db2 eval {
|
||||
INSERT INTO t1 VALUES(99);
|
||||
DELETE FROM t1 WHERE x=2;
|
||||
}
|
||||
db1 eval {
|
||||
SELECT x FROM t1 ORDER BY x;
|
||||
}
|
||||
} {1 3 99}
|
||||
do_test shared-$av-16.6 {
|
||||
file exists test.db
|
||||
} {0} ;# Verify that the database is in-memory
|
||||
|
||||
# Shared cache on named memory databases with different names.
|
||||
#
|
||||
do_test shared-$av-16.7 {
|
||||
db1 close
|
||||
db2 close
|
||||
forcedelete test1.db test2.db
|
||||
sqlite3 db1 file:test1.db?mode=memory -uri 1
|
||||
sqlite3 db2 file:test2.db?mode=memory -uri 1
|
||||
db1 eval {
|
||||
CREATE TABLE t1(x); INSERT INTO t1 VALUES(1),(2),(3);
|
||||
}
|
||||
catchsql {
|
||||
SELECT x FROM t1 ORDER BY x;
|
||||
} db2
|
||||
} {1 {no such table: t1}}
|
||||
do_test shared-$av-16.8 {
|
||||
file exists test1.db
|
||||
} {0} ;# Verify that the database is in-memory
|
||||
|
||||
|
||||
db1 close
|
||||
db2 close
|
||||
|
||||
} ;# end of autovacuum on/off loop
|
||||
|
||||
sqlite3_enable_shared_cache $::enable_shared_cache
|
||||
finish_test
|
||||
|
||||
Reference in New Issue
Block a user