database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Have fts5 tables delay initializing the tokenizer until it is first used in all cases where the tokenizer is not "trigram".

Browse Source 
FossilOrigin-Name: ca4fdcb8ae95d2a61236b949f852d2bf25ea2dbbff7eedafbd8eb84e8fd96687
This commit is contained in:
dan
2024-05-14 17:16:09 +00:00
parent 12b205c637
commit 32ca0dbcdf
8 changed files with 55 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ext/fts5/fts5Int.h
View File

@ -868,6 +868,7 @@ int sqlite3Fts5TokenizerPattern(
int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
Fts5Tokenizer *pTok
);
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/

2
ext/fts5/fts5_expr.c
View File

@ -324,7 +324,7 @@ int sqlite3Fts5ExprNew(
}
sqlite3_free(sParse.apPhrase);
*pzErr = sParse.zErr;
if( 0==*pzErr ) *pzErr = sParse.zErr;
return sParse.rc;
}

10
ext/fts5/fts5_main.c
View File

@ -380,7 +380,7 @@ static int fts5InitVtab(
pConfig->pzErrmsg = pzErr;
pTab->p.pConfig = pConfig;
pTab->pGlobal = pGlobal;
if( bCreate ){
if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
rc = sqlite3Fts5LoadTokenizer(pConfig);
}
}
@ -586,14 +586,6 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
return SQLITE_ERROR;
}
if( pConfig->t.pTok==0 ){
int rc;
pConfig->pzErrmsg = &pVTab->zErrMsg;
rc = sqlite3Fts5LoadTokenizer(pConfig);
pConfig->pzErrmsg = 0;
if( rc!=SQLITE_OK ) return rc;
}
idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1);
if( idxStr==0 ) return SQLITE_NOMEM;
pInfo->idxStr = idxStr;

10
ext/fts5/fts5_tokenize.c
View File

@ -1428,6 +1428,16 @@ int sqlite3Fts5TokenizerPattern(
return FTS5_PATTERN_NONE;
}
/*
** Return true if the tokenizer described by p->azArg[] is the trigram
** tokenizer. This tokenizer needs to be loaded before xBestIndex is
** called for the first time in order to correctly handle LIKE/GLOB.
*/
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
}
/*
** Register all built-in tokenizers with FTS5.
*/

2
ext/fts5/fts5_vocab.c
View File

@ -365,7 +365,7 @@ static int fts5VocabOpenMethod(
if( rc==SQLITE_OK ){
pVTab->zErrMsg = sqlite3_mprintf(
"no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
);
);
rc = SQLITE_ERROR;
}
}else{

31
ext/fts5/test/fts5tokenizer.test
View File

@ -315,7 +315,7 @@ db close
sqlite3 db test.db
do_catchsql_test 10.2 {
SELECT * FROM x1
SELECT * FROM x1('abc');
} {1 {error in tokenizer constructor}}
do_catchsql_test 10.3 {
@ -333,7 +333,7 @@ db close
sqlite3 db test.db
do_catchsql_test 10.5 {
SELECT * FROM x1
SELECT * FROM x1('abc');
} {1 {no such tokenizer: nosuch}}
do_catchsql_test 10.6 {
INSERT INTO x1 VALUES('abc');
@ -344,4 +344,31 @@ do_execsql_test 10.7 {
SELECT * FROM sqlite_schema;
}
reset_db
do_execsql_test 10.8 {
CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61);
INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c');
CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row);
PRAGMA writable_schema = 1;
UPDATE sqlite_schema
SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);'
WHERE name = 'x1';
}
do_execsql_test 10.9 {
SELECT * FROM x1v
} {
a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
}
db close
sqlite3 db test.db
do_execsql_test 10.10 {
SELECT * FROM x1v
} {
a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1
}
finish_test