database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-01-06 08:01:16 +03:00
Code Activity

Merge all the latests trunk enhancements into the win-dupe-crt-fio branch.

Browse Source 
FossilOrigin-Name: 2d52db98f47fbcda0622c034b21c2fb19bf4345b88c5c565ae9e6f7128642e43
This commit is contained in:
drh
2024-08-27 13:57:43 +00:00
parent 3a39c44c21 c013fa50db
commit fcd65efcf1
492 changed files with 24019 additions and 6220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
ext/consio/console_io.c
View File

@@ -29,6 +29,9 @@
#ifndef HAVE_CONSOLE_IO_H
# include "console_io.h"
#endif
#if defined(_MSC_VER)
# pragma warning(disable : 4204)
#endif
#ifndef SQLITE_CIO_NO_TRANSLATE
# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
@@ -50,11 +53,6 @@
# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
#endif
#if CIO_WIN_WC_XLATE
/* Character used to represent a known-incomplete UTF-8 char group (<28>) */
static WCHAR cBadGroup = 0xfffd;
#endif
#if CIO_WIN_WC_XLATE
static HANDLE handleOfFile(FILE *pf){
int fileDesc = _fileno(pf);
@@ -127,6 +125,10 @@ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
# endif
}
# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4)
# endif
# if CIO_WIN_WC_XLATE
/* Define console modes for use with the Windows Console API. */
# define SHELL_CONI_MODE \
@@ -755,4 +757,8 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
}
#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
#if defined(_MSC_VER)
# pragma warning(default : 4204)
#endif
#undef SHELL_INVALID_FILE_PTR

8
ext/expert/expert1.test
View File

@@ -8,6 +8,7 @@
# May you share freely, never taking more than you give.
#
#***********************************************************************
# TESTRUNNER: shell
#
# The focus of this file is testing the CLI shell tool. Specifically,
# the ".recommend" command.
@@ -464,6 +465,13 @@ do_execsql_test 5.3 {
t2 t2_idx_0001295b {100 20 5}
}
do_catchsql_test 5.4 {
SELECT sqlite_expert_rem(123, 123);
} {1 {no such function: sqlite_expert_rem}}
do_catchsql_test 5.5 {
SELECT sqlite_expert_sample();
} {1 {no such function: sqlite_expert_sample}}
if 0 {
do_test expert1-6.0 {
catchcmd :memory: {

22
ext/expert/sqlite3expert.c
View File

@@ -626,7 +626,7 @@ static int expertFilter(
pCsr->pData = 0;
if( rc==SQLITE_OK ){
rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
"SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
"SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName
);
}
@@ -1500,7 +1500,7 @@ struct IdxRemCtx {
};
/*
** Implementation of scalar function rem().
** Implementation of scalar function sqlite_expert_rem().
*/
static void idxRemFunc(
sqlite3_context *pCtx,
@@ -1513,7 +1513,7 @@ static void idxRemFunc(
assert( argc==2 );
iSlot = sqlite3_value_int(argv[0]);
assert( iSlot<=p->nSlot );
assert( iSlot<p->nSlot );
pSlot = &p->aSlot[iSlot];
switch( pSlot->eType ){
@@ -1624,7 +1624,8 @@ static int idxPopulateOneStat1(
const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
zCols = idxAppendText(&rc, zCols,
"%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
"%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s",
zComma, zName, nCol, zName, zColl
);
zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
}
@@ -1757,13 +1758,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
if( rc==SQLITE_OK ){
sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
rc = sqlite3_create_function(
dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
rc = sqlite3_create_function(dbrem, "sqlite_expert_rem",
2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(
p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
rc = sqlite3_create_function(p->db, "sqlite_expert_sample",
0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
);
}
@@ -1815,6 +1816,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0);
}
sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0);
sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0);
sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
return rc;
}
@@ -1948,7 +1952,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
sqlite3_stmt *pSql = 0;
rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg,
"SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'"
" AND sql NOT LIKE 'CREATE VIRTUAL %%'"
" AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid"
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
const char *zSql = (const char*)sqlite3_column_text(pSql, 0);

10
ext/expert/test_expert.c
View File

@@ -16,15 +16,7 @@
#include "sqlite3expert.h"
#include <assert.h>
#include <string.h>
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include "tclsqlite.h"
#ifndef SQLITE_OMIT_VIRTUALTABLE

37
ext/fts3/fts3.c
View File

@@ -4006,7 +4006,7 @@ static int fts3ShadowName(const char *zName){
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
static int fts3Integrity(
static int fts3IntegrityMethod(
sqlite3_vtab *pVtab, /* The virtual table to be checked */
const char *zSchema, /* Name of schema in which pVtab lives */
const char *zTabname, /* Name of the pVTab table */
@@ -4014,31 +4014,24 @@ static int fts3Integrity(
char **pzErr /* Write error message here */
){
Fts3Table *p = (Fts3Table*)pVtab;
char *zSql;
int rc;
char *zErr = 0;
int rc = SQLITE_OK;
int bOk = 0;
assert( pzErr!=0 );
assert( *pzErr==0 );
UNUSED_PARAMETER(isQuick);
zSql = sqlite3_mprintf(
"INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
zSchema, zTabname, zTabname);
if( zSql==0 ){
return SQLITE_NOMEM;
}
rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr);
sqlite3_free(zSql);
if( (rc&0xff)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
p->bFts4 ? 4 : 3, zSchema, zTabname);
}else if( rc!=SQLITE_OK ){
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
assert( rc!=SQLITE_CORRUPT_VTAB );
if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS%d table %s.%s: %s",
p->bFts4 ? 4 : 3, zSchema, zTabname, zErr);
p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
if( *pzErr ) rc = SQLITE_OK;
}else if( rc==SQLITE_OK && bOk==0 ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
p->bFts4 ? 4 : 3, zSchema, zTabname);
if( *pzErr==0 ) rc = SQLITE_NOMEM;
}
sqlite3_free(zErr);
return SQLITE_OK;
sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -4068,7 +4061,7 @@ static const sqlite3_module fts3Module = {
/* xRelease */ fts3ReleaseMethod,
/* xRollbackTo */ fts3RollbackToMethod,
/* xShadowName */ fts3ShadowName,
/* xIntegrity */ fts3Integrity,
/* xIntegrity */ fts3IntegrityMethod,
};
/*

2
ext/fts3/fts3Int.h
View File

@@ -653,5 +653,7 @@ int sqlite3FtsUnicodeIsdiacritic(int);
int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */

3
ext/fts3/fts3_snippet.c
View File

@@ -398,6 +398,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){
return 1;
}
assert( pIter->nSnippet>=0 );
pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
for(i=0; i<pIter->nPhrase; i++){
SnippetPhrase *pPhrase = &pIter->aPhrase[i];
@@ -446,7 +447,7 @@ static void fts3SnippetDetails(
}
mCover |= mPhrase;
for(j=0; j<pPhrase->nToken; j++){
for(j=0; j<pPhrase->nToken && j<pIter->nSnippet; j++){
mHighlight |= (mPos>>j);
}

16
ext/fts3/fts3_term.c
View File

@@ -78,6 +78,8 @@ static int fts3termConnectMethod(
iIndex = atoi(argv[4]);
argc--;
}
*ppVtab = 0;
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
@@ -95,12 +97,17 @@ static int fts3termConnectMethod(
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
p = (Fts3termTable *)sqlite3_malloc64(nByte);
nByte = sizeof(Fts3termTable);
p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, (size_t)nByte);
p->pFts3Tab = (Fts3Table *)&p[1];
p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero(
sizeof(Fts3Table) + nDb + nFts3 + 2
);
if( p->pFts3Tab==0 ){
sqlite3_free(p);
return SQLITE_NOMEM;
}
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
@@ -130,6 +137,7 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
sqlite3_finalize(pFts3->aStmt[i]);
}
sqlite3_free(pFts3->zSegmentsTbl);
sqlite3_free(pFts3);
sqlite3_free(p);
return SQLITE_OK;
}

29
ext/fts3/fts3_test.c
View File

@@ -18,14 +18,7 @@
** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include "tclsqlite.h"
#include <string.h>
#include <assert.h>
@@ -167,7 +160,8 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
Tcl_Obj *pPhrasecount = 0;
Tcl_Obj **apExprToken;
int nExprToken;
Tcl_Size nExprToken;
Tcl_Size nn;
UNUSED_PARAMETER(clientData);
@@ -201,23 +195,25 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
}
}
rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
doc.nToken = (int)nn;
if( rc!=TCL_OK ) goto near_match_out;
doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
for(ii=0; ii<doc.nToken; ii++){
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
doc.aToken[ii].n = (int)nn;
}
rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
if( rc!=TCL_OK ) goto near_match_out;
nPhrase = (nExprToken + 1) / 2;
nPhrase = (int)(nExprToken + 1) / 2;
aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
for(ii=0; ii<nPhrase; ii++){
Tcl_Obj *pPhrase = apExprToken[ii*2];
Tcl_Obj **apToken;
int nToken;
Tcl_Size nToken;
int jj;
rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
@@ -227,11 +223,12 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
rc = TCL_ERROR;
goto near_match_out;
}
for(jj=0; jj<nToken; jj++){
for(jj=0; jj<(int)nToken; jj++){
NearToken *pT = &aPhrase[ii].aToken[jj];
pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
pT->n = (int)nn;
}
aPhrase[ii].nToken = nToken;
aPhrase[ii].nToken = (int)nToken;
}
for(ii=1; ii<nPhrase; ii++){
Tcl_Obj *pNear = apExprToken[2*ii-1];

6
ext/fts3/fts3_tokenizer.c
View File

@@ -226,11 +226,7 @@ int sqlite3Fts3InitTokenizer(
#ifdef SQLITE_TEST
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
#endif
#include "tclsqlite.h"
#include <string.h>
/*

11
ext/fts3/fts3_write.c
View File

@@ -5294,7 +5294,7 @@ static u64 fts3ChecksumIndex(
** If an error occurs (e.g. an OOM or IO error), return an SQLite error
** code. The final value of *pbOk is undefined in this case.
*/
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int rc = SQLITE_OK; /* Return code */
u64 cksum1 = 0; /* Checksum based on FTS index contents */
u64 cksum2 = 0; /* Checksum based on %_content contents */
@@ -5372,7 +5372,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
sqlite3_finalize(pStmt);
}
*pbOk = (cksum1==cksum2);
if( rc==SQLITE_CORRUPT_VTAB ){
rc = SQLITE_OK;
*pbOk = 0;
}else{
*pbOk = (rc==SQLITE_OK && cksum1==cksum2);
}
return rc;
}
@@ -5412,7 +5417,7 @@ static int fts3DoIntegrityCheck(
){
int rc;
int bOk = 0;
rc = fts3IntegrityCheck(p, &bOk);
rc = sqlite3Fts3IntegrityCheck(p, &bOk);
if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
return rc;
}

3
ext/fts3/unicode/mkunicode.tcl
View File

@@ -628,6 +628,9 @@ proc print_categories {lMap} {
$caseP
$caseS
$caseZ
default:
return 1;
}
return 0;
}

7
ext/fts5/extract_api_docs.tcl
View File

@@ -82,7 +82,7 @@ proc get_struct_docs {data names} {
set current_doc ""
}
set subject n/a
regexp {^ *([[:alpha:]]*)} $line -> subject
regexp {^ *([[:alnum:]_]*)} $line -> subject
if {[lsearch $names $subject]>=0} {
set current_header $subject
} else {
@@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} {
append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
continue
}
if {[regexp {FTS5_TOKENIZER} $line]} {
set line </dl><p>
}
if {[regexp {SYNONYM SUPPORT} $line]} {
set line "</dl><h3>Synonym Support</h3>"
set line "<h3>Synonym Support</h3>"
}
if {[string trim $line] == ""} {
append res "<p>\n"

127
ext/fts5/fts5.h
View File

@@ -55,8 +55,8 @@ struct Fts5PhraseIter {
** EXTENSION API FUNCTIONS
**
** xUserData(pFts):
** Return a copy of the context pointer the extension function was
** registered with.
** Return a copy of the pUserData pointer passed to the xCreateFunction()
** API when the extension function was registered.
**
** xColumnTotalSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
@@ -238,6 +238,10 @@ struct Fts5PhraseIter {
** (i.e. if it is a contentless table), then this API always iterates
** through an empty set (all calls to xPhraseFirst() set iCol to -1).
**
** In all cases, matches are visited in (column ASC, offset ASC) order.
** i.e. all those in column 0, sorted by offset, followed by those in
** column 1, etc.
**
** xPhraseNext()
** See xPhraseFirst above.
**
@@ -304,9 +308,32 @@ struct Fts5PhraseIter {
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
**
** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
** If parameter iCol is less than zero, or greater than or equal to the
** number of columns in the table, SQLITE_RANGE is returned.
**
** Otherwise, this function attempts to retrieve the locale associated
** with column iCol of the current row. Usually, there is no associated
** locale, and output parameters (*pzLocale) and (*pnLocale) are set
** to NULL and 0, respectively. However, if the fts5_locale() function
** was used to associate a locale with the value when it was inserted
** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
** buffer containing the name of the locale in utf-8 encoding. (*pnLocale)
** is set to the size in bytes of the buffer, not including the
** nul-terminator.
**
** If successful, SQLITE_OK is returned. Or, if an error occurs, an
** SQLite error code is returned. The final value of the output parameters
** is undefined in this case.
**
** xTokenize_v2:
** Tokenize text using the tokenizer belonging to the FTS5 table. This
** API is the same as the xTokenize() API, except that it allows a tokenizer
** locale to be specified.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 3 */
int iVersion; /* Currently always set to 4 */
void *(*xUserData)(Fts5Context*);
@@ -348,6 +375,15 @@ struct Fts5ExtensionApi {
const char **ppToken, int *pnToken
);
int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
/* Below this point are iVersion>=4 only */
int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xTokenize_v2)(Fts5Context*,
const char *pText, int nText, /* Text to tokenize */
const char *pLocale, int nLocale, /* Locale to pass to tokenizer */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
};
/*
@@ -360,6 +396,7 @@ struct Fts5ExtensionApi {
** Applications may also register custom tokenizer types. A tokenizer
** is registered by providing fts5 with a populated instance of the
** following structure. All structure methods must be defined, setting
**
** any member of the fts5_tokenizer struct to NULL leads to undefined
** behaviour. The structure methods are expected to function as follows:
**
@@ -368,7 +405,7 @@ struct Fts5ExtensionApi {
** A tokenizer instance is required to actually tokenize text.
**
** The first argument passed to this function is a copy of the (void*)
** pointer provided by the application when the fts5_tokenizer object
** pointer provided by the application when the fts5_tokenizer_v2 object
** was registered with FTS5 (the third argument to xCreateTokenizer()).
** The second and third arguments are an array of nul-terminated strings
** containing the tokenizer arguments, if any, specified following the
@@ -392,7 +429,7 @@ struct Fts5ExtensionApi {
** argument passed to this function is a pointer to an Fts5Tokenizer object
** returned by an earlier call to xCreate().
**
** The second argument indicates the reason that FTS5 is requesting
** The third argument indicates the reason that FTS5 is requesting
** tokenization of the supplied text. This is always one of the following
** four values:
**
@@ -416,6 +453,13 @@ struct Fts5ExtensionApi {
** on a columnsize=0 database.
** </ul>
**
** The sixth and seventh arguments passed to xTokenize() - pLocale and
** nLocale - are a pointer to a buffer containing the locale to use for
** tokenization (e.g. "en_US") and its size in bytes, respectively. The
** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
** which case nLocale is always 0) to indicate that the tokenizer should
** use its default locale.
**
** For each token in the input string, the supplied callback xToken() must
** be invoked. The first argument to it should be a copy of the pointer
** passed as the second argument to xTokenize(). The third and fourth
@@ -439,6 +483,30 @@ struct Fts5ExtensionApi {
** may abandon the tokenization and return any error code other than
** SQLITE_OK or SQLITE_DONE.
**
** If the tokenizer is registered using an fts5_tokenizer_v2 object,
** then the xTokenize() method has two additional arguments - pLocale
** and nLocale. These specify the locale that the tokenizer should use
** for the current request. If pLocale and nLocale are both 0, then the
** tokenizer should use its default locale. Otherwise, pLocale points to
** an nLocale byte buffer containing the name of the locale to use as utf-8
** text. pLocale is not nul-terminated.
**
** FTS5_TOKENIZER
**
** There is also an fts5_tokenizer object. This is an older, deprecated,
** version of fts5_tokenizer_v2. It is similar except that:
**
** <ul>
** <li> There is no "iVersion" field, and
** <li> The xTokenize() method does not take a locale argument.
** </ul>
**
** Legacy fts5_tokenizer tokenizers must be registered using the
** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
**
** Tokenizer implementations registered using either API may be retrieved
** using both xFindTokenizer() and xFindTokenizer_v2().
**
** SYNONYM SUPPORT
**
** Custom tokenizers may also support synonyms. Consider a case in which a
@@ -547,6 +615,33 @@ struct Fts5ExtensionApi {
** inefficient.
*/
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
struct fts5_tokenizer_v2 {
int iVersion; /* Currently always 2 */
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
void (*xDelete)(Fts5Tokenizer*);
int (*xTokenize)(Fts5Tokenizer*,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
const char *pLocale, int nLocale,
int (*xToken)(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
)
);
};
/*
** New code should use the fts5_tokenizer_v2 type to define tokenizer
** implementations. The following type is included for legacy applications
** that still use it.
*/
typedef struct fts5_tokenizer fts5_tokenizer;
struct fts5_tokenizer {
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
@@ -566,6 +661,7 @@ struct fts5_tokenizer {
);
};
/* Flags that may be passed as the third argument to xTokenize() */
#define FTS5_TOKENIZE_QUERY 0x0001
#define FTS5_TOKENIZE_PREFIX 0x0002
@@ -585,7 +681,7 @@ struct fts5_tokenizer {
*/
typedef struct fts5_api fts5_api;
struct fts5_api {
int iVersion; /* Currently always set to 2 */
int iVersion; /* Currently always set to 3 */
/* Create a new tokenizer */
int (*xCreateTokenizer)(
@@ -612,6 +708,25 @@ struct fts5_api {
fts5_extension_function xFunction,
void (*xDestroy)(void*)
);
/* APIs below this point are only available if iVersion>=3 */
/* Create a new tokenizer */
int (*xCreateTokenizer_v2)(
fts5_api *pApi,
const char *zName,
void *pUserData,
fts5_tokenizer_v2 *pTokenizer,
void (*xDestroy)(void*)
);
/* Find an existing tokenizer */
int (*xFindTokenizer_v2)(
fts5_api *pApi,
const char *zName,
void **ppUserData,
fts5_tokenizer_v2 **ppTokenizer
);
};
/*

66
ext/fts5/fts5Int.h
View File

@@ -59,6 +59,22 @@ typedef sqlite3_uint64 u64;
# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
/* The uptr type is an unsigned integer large enough to hold a pointer
*/
#if defined(HAVE_STDINT_H)
typedef uintptr_t uptr;
#elif SQLITE_PTRSIZE==4
typedef u32 uptr;
#else
typedef u64 uptr;
#endif
#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0)
#else
# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0)
#endif
#endif
/* Truncate very long tokens to this many bytes. Hard limit is
@@ -142,6 +158,18 @@ struct Fts5Colset {
*/
typedef struct Fts5Config Fts5Config;
typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
struct Fts5TokenizerConfig {
Fts5Tokenizer *pTok;
fts5_tokenizer_v2 *pApi2;
fts5_tokenizer *pApi1;
const char **azArg;
int nArg;
int ePattern; /* FTS_PATTERN_XXX constant */
const char *pLocale; /* Current locale to use */
int nLocale; /* Size of pLocale in bytes */
};
/*
** An instance of the following structure encodes all information that can
@@ -181,9 +209,12 @@ typedef struct Fts5Config Fts5Config;
**
** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
**
** bLocale:
** Set to true if locale=1 was specified when the table was created.
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
Fts5Global *pGlobal; /* Global fts5 object for handle db */
char *zDb; /* Database holding FTS index (e.g. "main") */
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
@@ -197,12 +228,12 @@ struct Fts5Config {
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int bTokendata; /* "tokendata=" option value (dflt==0) */
int bLocale; /* "locale=" option value (dflt==0) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
Fts5TokenizerConfig t;
int bLock; /* True when table is preparing statement */
int ePattern; /* FTS_PATTERN_XXX constant */
/* Values loaded from the %_config table */
int iVersion; /* fts5 file format 'version' */
@@ -268,6 +299,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
@@ -312,7 +345,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
typedef struct Fts5PoslistReader Fts5PoslistReader;
@@ -597,18 +630,23 @@ struct Fts5Table {
Fts5Index *pIndex; /* Full-text index */
};
int sqlite3Fts5GetTokenizer(
Fts5Global*,
const char **azArg,
int nArg,
Fts5Config*,
char **pzErr
);
int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
int sqlite3Fts5FlushToDisk(Fts5Table*);
int sqlite3Fts5ExtractText(
Fts5Config *pConfig,
sqlite3_value *pVal, /* Value to extract text from */
int bContent, /* Loaded from content table */
int *pbResetTokenizer, /* OUT: True if ClearLocale() required */
const char **ppText, /* OUT: Pointer to text buffer */
int *pnText /* OUT: Size of (*ppText) in bytes */
);
void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
/*
** End of interface to code in fts5.c.
**************************************************************************/
@@ -688,7 +726,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
@@ -714,6 +752,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);
void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/
@@ -866,6 +907,7 @@ int sqlite3Fts5TokenizerPattern(
int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
Fts5Tokenizer *pTok
);
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/

82
ext/fts5/fts5_aux.c
View File

@@ -226,6 +226,7 @@ static int fts5HighlightCb(
return rc;
}
/*
** Implementation of highlight() function.
*/
@@ -256,12 +257,19 @@ static void fts5HighlightFunction(
sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
rc = SQLITE_OK;
}else if( ctx.zIn ){
const char *pLoc = 0; /* Locale of column iCol */
int nLoc = 0; /* Size of pLoc in bytes */
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize_v2(
pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -458,6 +466,8 @@ static void fts5SnippetFunction(
memset(&sFinder, 0, sizeof(Fts5SFinder));
for(i=0; i<nCol; i++){
if( iCol<0 || iCol==i ){
const char *pLoc = 0; /* Locale of column iCol */
int nLoc = 0; /* Size of pLoc in bytes */
int nDoc;
int nDocsize;
int ii;
@@ -465,8 +475,10 @@ static void fts5SnippetFunction(
sFinder.nFirst = 0;
rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xTokenize(pFts,
sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
if( rc!=SQLITE_OK ) break;
rc = pApi->xTokenize_v2(pFts,
sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
);
if( rc!=SQLITE_OK ) break;
rc = pApi->xColumnSize(pFts, i, &nDocsize);
@@ -524,6 +536,9 @@ static void fts5SnippetFunction(
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( ctx.zIn ){
const char *pLoc = 0; /* Locale of column iBestCol */
int nLoc = 0; /* Bytes in pLoc */
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
}
@@ -542,7 +557,12 @@ static void fts5SnippetFunction(
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize_v2(
pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
);
}
if( ctx.bOpen ){
fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -726,6 +746,53 @@ static void fts5Bm25Function(
}
}
/*
** Implementation of fts5_get_locale() function.
*/
static void fts5GetLocaleFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
int iCol = 0;
int eType = 0;
int rc = SQLITE_OK;
const char *zLocale = 0;
int nLocale = 0;
/* xColumnLocale() must be available */
assert( pApi->iVersion>=4 );
if( nVal!=1 ){
const char *z = "wrong number of arguments to function fts5_get_locale()";
sqlite3_result_error(pCtx, z, -1);
return;
}
eType = sqlite3_value_numeric_type(apVal[0]);
if( eType!=SQLITE_INTEGER ){
const char *z = "non-integer argument passed to function fts5_get_locale()";
sqlite3_result_error(pCtx, z, -1);
return;
}
iCol = sqlite3_value_int(apVal[0]);
if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
sqlite3_result_error_code(pCtx, SQLITE_RANGE);
return;
}
rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
return;
}
sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT);
}
int sqlite3Fts5AuxInit(fts5_api *pApi){
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
@@ -733,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
fts5_extension_function xFunc;/* Callback function */
void (*xDestroy)(void*); /* Destructor function */
} aBuiltin [] = {
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
{ "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */

115
ext/fts5/fts5_config.c
View File

@@ -234,7 +234,6 @@ static int fts5ConfigSetEnum(
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Global *pGlobal,
Fts5Config *pConfig, /* Configuration object to update */
const char *zCmd, /* Special command to parse */
const char *zArg, /* Argument to parse */
@@ -298,12 +297,11 @@ static int fts5ConfigParseSpecial(
if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
const char *p = (const char*)zArg;
sqlite3_int64 nArg = strlen(zArg) + 1;
char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
char *pSpace = pDel;
char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
if( azArg && pSpace ){
if( pConfig->pTok ){
if( azArg ){
char *pSpace = (char*)&azArg[nArg];
if( pConfig->t.azArg ){
*pzErr = sqlite3_mprintf("multiple tokenize=... directives");
rc = SQLITE_ERROR;
}else{
@@ -326,16 +324,14 @@ static int fts5ConfigParseSpecial(
*pzErr = sqlite3_mprintf("parse error in tokenize directive");
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, (int)nArg, pConfig,
pzErr
);
pConfig->t.azArg = (const char**)azArg;
pConfig->t.nArg = nArg;
azArg = 0;
}
}
}
sqlite3_free(azArg);
sqlite3_free(pDel);
return rc;
}
@@ -384,6 +380,16 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed locale=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bLocale = (zArg[0]=='1');
}
return rc;
}
if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
const Fts5Enum aDetail[] = {
{ "none", FTS5_DETAIL_NONE },
@@ -412,16 +418,6 @@ static int fts5ConfigParseSpecial(
return SQLITE_ERROR;
}
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
}
/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
@@ -554,6 +550,7 @@ int sqlite3Fts5ConfigParse(
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
memset(pRet, 0, sizeof(Fts5Config));
pRet->pGlobal = pGlobal;
pRet->db = db;
pRet->iCookie = -1;
@@ -602,7 +599,7 @@ int sqlite3Fts5ConfigParse(
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pGlobal, pRet,
rc = fts5ConfigParseSpecial(pRet,
ALWAYS(zOne)?zOne:"",
zTwo?zTwo:"",
pzErr
@@ -640,13 +637,6 @@ int sqlite3Fts5ConfigParse(
rc = SQLITE_ERROR;
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
const char *zTail = 0;
@@ -688,9 +678,14 @@ int sqlite3Fts5ConfigParse(
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->pTok ){
pConfig->pTokApi->xDelete(pConfig->pTok);
if( pConfig->t.pTok ){
if( pConfig->t.pApi1 ){
pConfig->t.pApi1->xDelete(pConfig->t.pTok);
}else{
pConfig->t.pApi2->xDelete(pConfig->t.pTok);
}
}
sqlite3_free((char*)pConfig->t.azArg);
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
@@ -765,10 +760,24 @@ int sqlite3Fts5Tokenize(
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
){
if( pText==0 ) return SQLITE_OK;
return pConfig->pTokApi->xTokenize(
pConfig->pTok, pCtx, flags, pText, nText, xToken
);
int rc = SQLITE_OK;
if( pText ){
if( pConfig->t.pTok==0 ){
rc = sqlite3Fts5LoadTokenizer(pConfig);
}
if( rc==SQLITE_OK ){
if( pConfig->t.pApi1 ){
rc = pConfig->t.pApi1->xTokenize(
pConfig->t.pTok, pCtx, flags, pText, nText, xToken
);
}else{
rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags,
pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
);
}
}
}
return rc;
}
/*
@@ -1022,13 +1031,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
&& iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
){
rc = SQLITE_ERROR;
if( pConfig->pzErrmsg ){
assert( 0==*pConfig->pzErrmsg );
*pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
"(found %d, expected %d or %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
);
}
sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
"(found %d, expected %d or %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
);
}else{
pConfig->iVersion = iVersion;
}
@@ -1038,3 +1044,26 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
}
return rc;
}
/*
** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer
** containing the error message created using printf() style formatting
** string zFmt and its trailing arguments.
*/
void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
va_list ap; /* ... printf arguments */
char *zMsg = 0;
va_start(ap, zFmt);
zMsg = sqlite3_vmprintf(zFmt, ap);
if( pConfig->pzErrmsg ){
assert( *pConfig->pzErrmsg==0 );
*pConfig->pzErrmsg = zMsg;
}else{
sqlite3_free(zMsg);
}
va_end(ap);
}

48
ext/fts5/fts5_expr.c
View File

@@ -54,7 +54,7 @@ struct Fts5Expr {
/*
** eType:
** Expression node type. Always one of:
** Expression node type. Usually one of:
**
** FTS5_AND (nChild, apChild valid)
** FTS5_OR (nChild, apChild valid)
@@ -62,6 +62,10 @@ struct Fts5Expr {
** FTS5_STRING (pNear valid)
** FTS5_TERM (pNear valid)
**
** An expression node with eType==0 may also exist. It always matches zero
** rows. This is created when a phrase containing no tokens is parsed.
** e.g. "".
**
** iHeight:
** Distance from this node to furthest leaf. This is always 0 for nodes
** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one
@@ -282,11 +286,12 @@ int sqlite3Fts5ExprNew(
}while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
assert( sParse.pExpr || sParse.rc!=SQLITE_OK );
assert_expr_depth_ok(sParse.rc, sParse.pExpr);
/* If the LHS of the MATCH expression was a user column, apply the
** implicit column-filter. */
if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){
int n = sizeof(Fts5Colset);
Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
if( pColset ){
@@ -303,15 +308,7 @@ int sqlite3Fts5ExprNew(
sParse.rc = SQLITE_NOMEM;
sqlite3Fts5ParseNodeFree(sParse.pExpr);
}else{
if( !sParse.pExpr ){
const int nByte = sizeof(Fts5ExprNode);
pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
if( pNew->pRoot ){
pNew->pRoot->bEof = 1;
}
}else{
pNew->pRoot = sParse.pExpr;
}
pNew->pRoot = sParse.pExpr;
pNew->pIndex = 0;
pNew->pConfig = pConfig;
pNew->apExprPhrase = sParse.apPhrase;
@@ -324,7 +321,11 @@ int sqlite3Fts5ExprNew(
}
sqlite3_free(sParse.apPhrase);
*pzErr = sParse.zErr;
if( 0==*pzErr ){
*pzErr = sParse.zErr;
}else{
sqlite3_free(sParse.zErr);
}
return sParse.rc;
}
@@ -1125,7 +1126,7 @@ static int fts5ExprNodeTest_STRING(
}
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
if( pIter->iRowid==iLast || pIter->bEof ) continue;
if( pIter->iRowid==iLast ) continue;
bMatch = 0;
if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
return rc;
@@ -1647,9 +1648,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5ExprNearset *pRet = 0;
if( pParse->rc==SQLITE_OK ){
if( pPhrase==0 ){
return pNear;
}
if( pNear==0 ){
sqlite3_int64 nByte;
nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
@@ -1871,6 +1869,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
}else if( sCtx.pPhrase->nTerm ){
sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
}
assert( pParse->apPhrase!=0 );
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
}
@@ -1890,7 +1889,7 @@ int sqlite3Fts5ExprClonePhrase(
Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */
Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */
if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
rc = SQLITE_RANGE;
}else{
pOrig = pExpr->apExprPhrase[iPhrase];
@@ -2258,6 +2257,9 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
}
}
/*
** Add pSub as a child of p.
*/
static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
int ii = p->nChild;
if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
@@ -2402,19 +2404,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
"fts5: %s queries are not supported (detail!=full)",
pNear->nPhrase==1 ? "phrase": "NEAR"
);
sqlite3_free(pRet);
sqlite3Fts5ParseNodeFree(pRet);
pRet = 0;
pNear = 0;
assert( pLeft==0 && pRight==0 );
}
}
}else{
assert( pNear==0 );
fts5ExprAddChildren(pRet, pLeft);
fts5ExprAddChildren(pRet, pRight);
pLeft = pRight = 0;
if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
sqlite3Fts5ParseError(pParse,
"fts5 expression tree is too large (maximum depth %d)",
SQLITE_FTS5_MAX_EXPR_DEPTH
);
sqlite3_free(pRet);
sqlite3Fts5ParseNodeFree(pRet);
pRet = 0;
}
}
@@ -2452,6 +2458,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
assert( pRight->eType==FTS5_STRING
|| pRight->eType==FTS5_TERM
|| pRight->eType==FTS5_EOF
|| (pRight->eType==FTS5_AND && pParse->bPhraseToAnd)
);
if( pLeft->eType==FTS5_AND ){
@@ -2465,6 +2472,8 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
);
if( pRight->eType==FTS5_EOF ){
assert( pParse->apPhrase!=0 );
assert( pParse->nPhrase>0 );
assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
sqlite3Fts5ParseNodeFree(pRight);
pRet = pLeft;
@@ -3097,6 +3106,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
pNode->iRowid = iRowid;
pNode->bEof = 0;
switch( pNode->eType ){
case 0:
case FTS5_TERM:
case FTS5_STRING:
return (pNode->pNear->apPhrase[0]->poslist.n>0);

17
ext/fts5/fts5_index.c
View File

@@ -831,11 +831,12 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
if( rc==SQLITE_OK ){
u8 *aOut = 0; /* Read blob data into this buffer */
int nByte = sqlite3_blob_bytes(p->pReader);
sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
int szData = (sizeof(Fts5Data) + 7) & ~7;
sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING;
pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
if( pRet ){
pRet->nn = nByte;
aOut = pRet->p = (u8*)&pRet[1];
aOut = pRet->p = (u8*)pRet + szData;
}else{
rc = SQLITE_NOMEM;
}
@@ -858,6 +859,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
}
assert( (pRet==0)==(p->rc!=SQLITE_OK) );
assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) );
return pRet;
}
@@ -6837,23 +6839,26 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){
int ii;
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
Fts5Index *pIndex = pIter->pIndex;
for(ii=0; ii<pT->nIter; ii++){
Fts5Iter *p = pT->apIter[ii];
if( p->base.bEof==0
&& (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowid<iFrom))
){
fts5MultiIterNext(p->pIndex, p, bFrom, iFrom);
fts5MultiIterNext(pIndex, p, bFrom, iFrom);
while( bFrom && p->base.bEof==0
&& p->base.iRowid<iFrom
&& p->pIndex->rc==SQLITE_OK
&& pIndex->rc==SQLITE_OK
){
fts5MultiIterNext(p->pIndex, p, 0, 0);
fts5MultiIterNext(pIndex, p, 0, 0);
}
}
}
fts5IterSetOutputsTokendata(pIter);
if( pIndex->rc==SQLITE_OK ){
fts5IterSetOutputsTokendata(pIter);
}
}
/*

1101
ext/fts5/fts5_main.c
View File

File diff suppressed because it is too large Load Diff

281
ext/fts5/fts5_storage.c
View File

@@ -16,13 +16,40 @@
#include "fts5Int.h"
/*
** pSavedRow:
** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it
** does a by-rowid lookup to retrieve a single row from the %_content
** table or equivalent external-content table/view.
**
** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
** values for a row being UPDATEd. In that case, the SQL statement is
** not reset and pSavedRow is set to point at it. This is so that the
** insert operation that follows the delete may access the original
** row values for any new values for which sqlite3_value_nochange() returns
** true. i.e. if the user executes:
**
** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
** ...
** UPDATE fts SET a=?, b=? WHERE rowid=?;
**
** then the value passed to the xUpdate() method of this table as the
** new.c value is an sqlite3_value_nochange() value. So in this case it
** must be read from the saved row stored in Fts5Storage.pSavedRow.
**
** This is necessary - using sqlite3_value_nochange() instead of just having
** SQLite pass the original value back via xUpdate() - so as not to discard
** any locale information associated with such values.
**
*/
struct Fts5Storage {
Fts5Config *pConfig;
Fts5Index *pIndex;
int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
i64 nTotalRow; /* Total number of rows in FTS table */
i64 *aTotalSize; /* Total sizes of each column */
sqlite3_stmt *aStmt[11];
sqlite3_stmt *pSavedRow;
sqlite3_stmt *aStmt[12];
};
@@ -36,14 +63,15 @@ struct Fts5Storage {
# error "FTS5_STMT_LOOKUP mismatch"
#endif
#define FTS5_STMT_INSERT_CONTENT 3
#define FTS5_STMT_REPLACE_CONTENT 4
#define FTS5_STMT_DELETE_CONTENT 5
#define FTS5_STMT_REPLACE_DOCSIZE 6
#define FTS5_STMT_DELETE_DOCSIZE 7
#define FTS5_STMT_LOOKUP_DOCSIZE 8
#define FTS5_STMT_REPLACE_CONFIG 9
#define FTS5_STMT_SCAN 10
#define FTS5_STMT_LOOKUP2 3
#define FTS5_STMT_INSERT_CONTENT 4
#define FTS5_STMT_REPLACE_CONTENT 5
#define FTS5_STMT_DELETE_CONTENT 6
#define FTS5_STMT_REPLACE_DOCSIZE 7
#define FTS5_STMT_DELETE_DOCSIZE 8
#define FTS5_STMT_LOOKUP_DOCSIZE 9
#define FTS5_STMT_REPLACE_CONFIG 10
#define FTS5_STMT_SCAN 11
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -73,6 +101,7 @@ static int fts5StorageGetStmt(
"SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
"SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
"SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
"SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
@@ -88,6 +117,8 @@ static int fts5StorageGetStmt(
Fts5Config *pC = p->pConfig;
char *zSql = 0;
assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
switch( eStmt ){
case FTS5_STMT_SCAN:
zSql = sqlite3_mprintf(azStmt[eStmt],
@@ -104,6 +135,7 @@ static int fts5StorageGetStmt(
break;
case FTS5_STMT_LOOKUP:
case FTS5_STMT_LOOKUP2:
zSql = sqlite3_mprintf(azStmt[eStmt],
pC->zContentExprlist, pC->zContent, pC->zContentRowid
);
@@ -150,7 +182,7 @@ static int fts5StorageGetStmt(
rc = SQLITE_NOMEM;
}else{
int f = SQLITE_PREPARE_PERSISTENT;
if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
p->pConfig->bLock++;
rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
p->pConfig->bLock--;
@@ -399,15 +431,49 @@ static int fts5StorageInsertCallback(
return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
}
/*
** This function is used as part of an UPDATE statement that modifies the
** rowid of a row. In that case, this function is called first to set
** Fts5Storage.pSavedRow to point to a statement that may be used to
** access the original values of the row being deleted - iDel.
**
** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
** It is not considered an error if row iDel does not exist. In this case
** pSavedRow is not set and SQLITE_OK returned.
*/
int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
int rc = SQLITE_OK;
sqlite3_stmt *pSeek = 0;
assert( p->pSavedRow==0 );
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
rc = sqlite3_reset(pSeek);
}else{
p->pSavedRow = pSeek;
}
}
return rc;
}
/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
**
** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
** the original values of the row being deleted. This is used by UPDATE
** statements.
*/
static int fts5StorageDeleteFromIndex(
Fts5Storage *p,
i64 iDel,
sqlite3_value **apVal
sqlite3_value **apVal,
int bSaveRow /* True to set pSavedRow */
){
Fts5Config *pConfig = p->pConfig;
sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
@@ -416,12 +482,21 @@ static int fts5StorageDeleteFromIndex(
int iCol;
Fts5InsertCtx ctx;
assert( bSaveRow==0 || apVal==0 );
assert( bSaveRow==0 || bSaveRow==1 );
assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
if( apVal==0 ){
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
return sqlite3_reset(pSeek);
if( p->pSavedRow && bSaveRow ){
pSeek = p->pSavedRow;
p->pSavedRow = 0;
}else{
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int64(pSeek, 1, iDel);
if( sqlite3_step(pSeek)!=SQLITE_ROW ){
return sqlite3_reset(pSeek);
}
}
}
@@ -429,26 +504,32 @@ static int fts5StorageDeleteFromIndex(
ctx.iCol = -1;
for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
if( pConfig->abUnindexed[iCol-1]==0 ){
const char *zText;
int nText;
sqlite3_value *pVal = 0;
const char *pText = 0;
int nText = 0;
int bReset = 0;
assert( pSeek==0 || apVal==0 );
assert( pSeek!=0 || apVal!=0 );
if( pSeek ){
zText = (const char*)sqlite3_column_text(pSeek, iCol);
nText = sqlite3_column_bytes(pSeek, iCol);
}else if( ALWAYS(apVal) ){
zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
nText = sqlite3_value_bytes(apVal[iCol-1]);
pVal = sqlite3_column_value(pSeek, iCol);
}else{
continue;
pVal = apVal[iCol-1];
}
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
zText, nText, (void*)&ctx, fts5StorageInsertCallback
rc = sqlite3Fts5ExtractText(
pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
if( p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
if( rc==SQLITE_OK ){
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
pText, nText, (void*)&ctx, fts5StorageInsertCallback
);
p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
}
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
}
}
@@ -458,11 +539,29 @@ static int fts5StorageDeleteFromIndex(
p->nTotalRow--;
}
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
if( rc==SQLITE_OK && bSaveRow ){
assert( p->pSavedRow==0 );
p->pSavedRow = pSeek;
}else{
rc2 = sqlite3_reset(pSeek);
if( rc==SQLITE_OK ) rc = rc2;
}
return rc;
}
/*
** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
** should be called by the xUpdate() method of the fts5 table before
** returning from any operation that may have set Fts5Storage.pSavedRow.
*/
void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
assert( pStorage->pSavedRow==0
|| pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2]
);
sqlite3_reset(pStorage->pSavedRow);
pStorage->pSavedRow = 0;
}
/*
** This function is called to process a DELETE on a contentless_delete=1
** table. It adds the tombstone required to delete the entry with rowid
@@ -519,12 +618,12 @@ static int fts5StorageInsertDocsize(
rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
sqlite3_bind_int64(pReplace, 3, iOrigin);
}
if( rc==SQLITE_OK ){
sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
sqlite3_bind_null(pReplace, 2);
}
}
if( rc==SQLITE_OK ){
sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
sqlite3_bind_null(pReplace, 2);
}
}
return rc;
@@ -578,7 +677,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
int sqlite3Fts5StorageDelete(
Fts5Storage *p, /* Storage object */
i64 iDel, /* Rowid to delete from table */
sqlite3_value **apVal, /* Optional - values to remove from index */
int bSaveRow /* If true, set pSavedRow for deleted row */
){
Fts5Config *pConfig = p->pConfig;
int rc;
sqlite3_stmt *pDel = 0;
@@ -595,7 +699,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
if( p->pConfig->bContentlessDelete ){
rc = fts5StorageContentlessDelete(p, iDel);
}else{
rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
}
}
@@ -684,14 +788,21 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -775,7 +886,31 @@ int sqlite3Fts5StorageContentInsert(
int i; /* Counter variable */
rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
rc = sqlite3_bind_value(pInsert, i, apVal[i]);
sqlite3_value *pVal = apVal[i];
if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
/* This is an UPDATE statement, and column (i-2) was not modified.
** Retrieve the value from Fts5Storage.pSavedRow instead. */
pVal = sqlite3_column_value(p->pSavedRow, i-1);
}else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
assert( pConfig->bLocale );
assert( i>1 );
if( pConfig->abUnindexed[i-2] ){
/* At attempt to insert an fts5_locale() value into an UNINDEXED
** column. Strip the locale away and just bind the text. */
const char *pText = 0;
int nText = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
}else{
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
assert( nBlob>4 );
sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
}
continue;
}
rc = sqlite3_bind_value(pInsert, i, pVal);
}
if( rc==SQLITE_OK ){
sqlite3_step(pInsert);
@@ -810,14 +945,24 @@ int sqlite3Fts5StorageIndexInsert(
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = apVal[ctx.iCol+2];
int bDisk = 0;
if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
bDisk = 1;
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
if( rc==SQLITE_OK ){
assert( bReset==0 || pConfig->bLocale );
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -988,14 +1133,22 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
int nText = sqlite3_column_bytes(pScan, i+1);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
zText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
rc = sqlite3Fts5ExtractText(pConfig,
sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;

309
ext/fts5/fts5_tcl.c
View File

@@ -14,14 +14,7 @@
#ifdef SQLITE_TEST
#if defined(INCLUDE_SQLITE_TCL_H)
# include "sqlite_tcl.h"
#else
# include "tcl.h"
# ifndef SQLITE_TCLAPI
# define SQLITE_TCLAPI
# endif
#endif
#include "tclsqlite.h"
#ifdef SQLITE_ENABLE_FTS5
@@ -247,6 +240,7 @@ static int SQLITE_TCLAPI xF5tApi(
{ "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */
{ "xInstToken", 2, "IDX ITERM" }, /* 19 */
{ "xColumnLocale", 1, "COL" }, /* 20 */
{ 0, 0, 0}
};
@@ -297,12 +291,12 @@ static int SQLITE_TCLAPI xF5tApi(
break;
}
CASE(3, "xTokenize") {
int nText;
Tcl_Size nText;
char *zText = Tcl_GetStringFromObj(objv[2], &nText);
F5tFunction ctx;
ctx.interp = interp;
ctx.pScript = objv[3];
rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
if( rc==SQLITE_OK ){
Tcl_ResetResult(interp);
}
@@ -535,6 +529,20 @@ static int SQLITE_TCLAPI xF5tApi(
break;
}
CASE(20, "xColumnLocale") {
const char *z = 0;
int n = 0;
int iCol;
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
return TCL_ERROR;
}
rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
if( rc==SQLITE_OK && z ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
}
break;
}
default:
assert( 0 );
break;
@@ -605,15 +613,16 @@ static void xF5tFunction(
sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
}else{
Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
int n;
const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
char c = zType[0];
if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
/* Only return a BLOB type if the Tcl variable is a bytearray and
** has no string representation. */
unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
Tcl_Size nn;
unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
}else if( c=='b' && strcmp(zType,"boolean")==0 ){
int n;
Tcl_GetIntFromObj(0, pVar, &n);
sqlite3_result_int(pCtx, n);
}else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -626,8 +635,9 @@ static void xF5tFunction(
Tcl_GetWideIntFromObj(0, pVar, &v);
sqlite3_result_int64(pCtx, v);
}else{
unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
Tcl_Size nn;
unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
}
}
}
@@ -720,7 +730,7 @@ static int SQLITE_TCLAPI f5tTokenize(
Tcl_Obj *CONST objv[]
){
char *zText;
int nText;
Tcl_Size nText;
sqlite3 *db = 0;
fts5_api *pApi = 0;
Fts5Tokenizer *pTok = 0;
@@ -729,7 +739,7 @@ static int SQLITE_TCLAPI f5tTokenize(
void *pUserdata;
int rc;
int nArg;
Tcl_Size nArg;
const char **azArg;
F5tTokenizeCtx ctx;
@@ -761,7 +771,7 @@ static int SQLITE_TCLAPI f5tTokenize(
return TCL_ERROR;
}
rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
return TCL_ERROR;
@@ -773,7 +783,7 @@ static int SQLITE_TCLAPI f5tTokenize(
ctx.pRet = pRet;
ctx.zInput = zText;
rc = tokenizer.xTokenize(
pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2
);
tokenizer.xDelete(pTok);
if( rc!=SQLITE_OK ){
@@ -801,18 +811,32 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance;
struct F5tTokenizerContext {
void *pCtx;
int (*xToken)(void*, int, const char*, int, int, int);
F5tTokenizerInstance *pInst;
};
struct F5tTokenizerModule {
Tcl_Interp *interp;
Tcl_Obj *pScript;
void *pParentCtx;
fts5_tokenizer_v2 parent_v2;
fts5_tokenizer parent;
F5tTokenizerContext *pContext;
};
/*
** zLocale:
** Within a call to xTokenize_v2(), pLocale/nLocale store the locale
** passed to the call by fts5. This can be retrieved by a Tcl tokenize
** script using [sqlite3_fts5_locale].
*/
struct F5tTokenizerInstance {
Tcl_Interp *interp;
Tcl_Obj *pScript;
F5tTokenizerModule *pModule;
Fts5Tokenizer *pParent;
F5tTokenizerContext *pContext;
const char *pLocale;
int nLocale;
};
static int f5tTokenizerCreate(
@@ -821,11 +845,20 @@ static int f5tTokenizerCreate(
int nArg,
Fts5Tokenizer **ppOut
){
Fts5Tokenizer *pParent = 0;
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
Tcl_Obj *pEval;
int rc = TCL_OK;
int i;
assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
if( pMod->parent_v2.xCreate ){
rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
}
if( pMod->parent.xCreate ){
rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
}
pEval = Tcl_DuplicateObj(pMod->pScript);
Tcl_IncrRefCount(pEval);
for(i=0; rc==TCL_OK && i<nArg; i++){
@@ -845,6 +878,8 @@ static int f5tTokenizerCreate(
pInst->interp = pMod->interp;
pInst->pScript = Tcl_GetObjResult(pMod->interp);
pInst->pContext = pMod->pContext;
pInst->pParent = pParent;
pInst->pModule = pMod;
Tcl_IncrRefCount(pInst->pScript);
*ppOut = (Fts5Tokenizer*)pInst;
}
@@ -855,11 +890,21 @@ static int f5tTokenizerCreate(
static void f5tTokenizerDelete(Fts5Tokenizer *p){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
Tcl_DecrRefCount(pInst->pScript);
ckfree((char *)pInst);
if( pInst ){
if( pInst->pParent ){
if( pInst->pModule->parent_v2.xDelete ){
pInst->pModule->parent_v2.xDelete(pInst->pParent);
}else{
pInst->pModule->parent.xDelete(pInst->pParent);
}
}
Tcl_DecrRefCount(pInst->pScript);
ckfree((char *)pInst);
}
}
static int f5tTokenizerTokenize(
static int f5tTokenizerReallyTokenize(
Fts5Tokenizer *p,
void *pCtx,
int flags,
@@ -867,6 +912,7 @@ static int f5tTokenizerTokenize(
int (*xToken)(void*, int, const char*, int, int, int)
){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
F5tTokenizerInstance *pOldInst = 0;
void *pOldCtx;
int (*xOldToken)(void*, int, const char*, int, int, int);
Tcl_Obj *pEval;
@@ -875,9 +921,11 @@ static int f5tTokenizerTokenize(
pOldCtx = pInst->pContext->pCtx;
xOldToken = pInst->pContext->xToken;
pOldInst = pInst->pContext->pInst;
pInst->pContext->pCtx = pCtx;
pInst->pContext->xToken = xToken;
pInst->pContext->pInst = pInst;
assert(
flags==FTS5_TOKENIZE_DOCUMENT
@@ -913,9 +961,105 @@ static int f5tTokenizerTokenize(
pInst->pContext->pCtx = pOldCtx;
pInst->pContext->xToken = xOldToken;
pInst->pContext->pInst = pOldInst;
return rc;
}
typedef struct CallbackCtx CallbackCtx;
struct CallbackCtx {
Fts5Tokenizer *p;
void *pCtx;
int flags;
int (*xToken)(void*, int, const char*, int, int, int);
};
static int f5tTokenizeCallback(
void *pCtx,
int tflags,
const char *z, int n,
int iStart, int iEnd
){
CallbackCtx *p = (CallbackCtx*)pCtx;
return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
}
static int f5tTokenizerTokenize_v2(
Fts5Tokenizer *p,
void *pCtx,
int flags,
const char *pText, int nText,
const char *pLoc, int nLoc,
int (*xToken)(void*, int, const char*, int, int, int)
){
int rc = SQLITE_OK;
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
pInst->pLocale = pLoc;
pInst->nLocale = nLoc;
if( pInst->pParent ){
CallbackCtx ctx;
ctx.p = p;
ctx.pCtx = pCtx;
ctx.flags = flags;
ctx.xToken = xToken;
if( pInst->pModule->parent_v2.xTokenize ){
rc = pInst->pModule->parent_v2.xTokenize(
pInst->pParent, (void*)&ctx, flags, pText, nText,
pLoc, nLoc, f5tTokenizeCallback
);
}else{
rc = pInst->pModule->parent.xTokenize(
pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
);
}
}else{
rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
}
pInst->pLocale = 0;
pInst->nLocale = 0;
return rc;
}
static int f5tTokenizerTokenize(
Fts5Tokenizer *p,
void *pCtx,
int flags,
const char *pText, int nText,
int (*xToken)(void*, int, const char*, int, int, int)
){
return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
}
/*
** sqlite3_fts5_locale
*/
static int SQLITE_TCLAPI f5tTokenizerLocale(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
if( objc!=1 ){
Tcl_WrongNumArgs(interp, 1, objv, "");
return TCL_ERROR;
}
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_locale may only be used by tokenizer callback", 0
);
return TCL_ERROR;
}
Tcl_SetObjResult(interp,
Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
);
return TCL_OK;
}
/*
** sqlite3_fts5_token ?-colocated? TEXT START END
*/
@@ -928,13 +1072,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
int iStart;
int iEnd;
int nToken;
Tcl_Size nToken;
int tflags = 0;
char *zToken;
int rc;
if( objc==5 ){
int nArg;
Tcl_Size nArg;
char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
tflags |= FTS5_TOKEN_COLOCATED;
@@ -959,7 +1103,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
return TCL_ERROR;
}
rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
@@ -1001,32 +1145,112 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
fts5_api *pApi;
char *zName;
Tcl_Obj *pScript;
fts5_tokenizer t;
F5tTokenizerModule *pMod;
int rc;
int rc = SQLITE_OK;
int bV2 = 0; /* True to use _v2 API */
int iVersion = 2; /* Value for _v2.iVersion */
const char *zParent = 0; /* Name of parent tokenizer, if any */
int ii = 0;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
if( objc<4 ){
Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
return TCL_ERROR;
}
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
/* Parse any options. Set stack variables bV2 and zParent. */
for(ii=1; ii<objc-3; ii++){
int iOpt = 0;
const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
return TCL_ERROR;
}
switch( iOpt ){
case 0: /* -v2 */ {
bV2 = 1;
break;
}
case 1: /* -parent */ {
ii++;
if( ii==objc-3 ){
Tcl_AppendResult(
interp, "option requires an argument: -parent", (char*)0
);
return TCL_ERROR;
}
zParent = Tcl_GetString(objv[ii]);
break;
}
case 2: /* -version */ {
ii++;
if( ii==objc-3 ){
Tcl_AppendResult(
interp, "option requires an argument: -version", (char*)0
);
return TCL_ERROR;
}
if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
return TCL_ERROR;
}
break;
}
default:
assert( 0 );
break;
}
}
if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
return TCL_ERROR;
}
zName = Tcl_GetString(objv[2]);
pScript = objv[3];
t.xCreate = f5tTokenizerCreate;
t.xTokenize = f5tTokenizerTokenize;
t.xDelete = f5tTokenizerDelete;
zName = Tcl_GetString(objv[objc-2]);
pScript = objv[objc-1];
pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
memset(pMod, 0, sizeof(F5tTokenizerModule));
pMod->interp = interp;
pMod->pScript = pScript;
pMod->pContext = pContext;
Tcl_IncrRefCount(pScript);
rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
pMod->pContext = pContext;
if( zParent ){
if( bV2 ){
fts5_tokenizer_v2 *pParent = 0;
rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
if( rc==SQLITE_OK ){
memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
pMod->parent_v2.xDelete(0);
}
}else{
rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
if( rc==SQLITE_OK ){
pMod->parent.xDelete(0);
}
}
}
if( rc==SQLITE_OK ){
void *pModCtx = (void*)pMod;
if( bV2==0 ){
fts5_tokenizer t;
t.xCreate = f5tTokenizerCreate;
t.xTokenize = f5tTokenizerTokenize;
t.xDelete = f5tTokenizerDelete;
rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
}else{
fts5_tokenizer_v2 t2;
memset(&t2, 0, sizeof(t2));
t2.iVersion = iVersion;
t2.xCreate = f5tTokenizerCreate;
t2.xTokenize = f5tTokenizerTokenize_v2;
t2.xDelete = f5tTokenizerDelete;
rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
}
}
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
Tcl_AppendResult(interp, (
bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
: "error in fts5_api.xCreateTokenizer()"
), 0);
return TCL_ERROR;
}
@@ -1083,7 +1307,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
Tcl_Obj *CONST objv[]
){
char *z;
int n;
Tcl_Size n;
unsigned int iVal;
int nSlot;
@@ -1096,7 +1320,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
}
z = Tcl_GetStringFromObj(objv[2], &n);
iVal = f5t_fts5HashKey(nSlot, z, n);
iVal = f5t_fts5HashKey(nSlot, z, (int)n);
Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
return TCL_OK;
}
@@ -1169,7 +1393,7 @@ struct OriginTextTokenizer {
*/
static void f5tOrigintextTokenizerDelete(void *pCtx){
OriginTextCtx *p = (OriginTextCtx*)pCtx;
ckfree(p);
ckfree((char*)p);
}
static int f5tOrigintextCreate(
@@ -1333,6 +1557,7 @@ int Fts5tcl_Init(Tcl_Interp *interp){
} aCmd[] = {
{ "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
{ "sqlite3_fts5_token", f5tTokenizerReturn, 1 },
{ "sqlite3_fts5_locale", f5tTokenizerLocale, 1 },
{ "sqlite3_fts5_tokenize", f5tTokenize, 0 },
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 },
{ "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 },

103
ext/fts5/fts5_tokenize.c
View File

@@ -386,7 +386,6 @@ static int fts5UnicodeCreate(
zCat = azArg[i+1];
}
}
if( rc==SQLITE_OK ){
rc = unicodeSetCategories(p, zCat);
}
@@ -416,7 +415,6 @@ static int fts5UnicodeCreate(
rc = SQLITE_ERROR;
}
}
}else{
rc = SQLITE_NOMEM;
}
@@ -555,7 +553,7 @@ static int fts5UnicodeTokenize(
typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
fts5_tokenizer tokenizer; /* Parent tokenizer module */
fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};
@@ -567,7 +565,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){
if( pTok ){
PorterTokenizer *p = (PorterTokenizer*)pTok;
if( p->pTokenizer ){
p->tokenizer.xDelete(p->pTokenizer);
p->tokenizer_v2.xDelete(p->pTokenizer);
}
sqlite3_free(p);
}
@@ -586,6 +584,7 @@ static int fts5PorterCreate(
PorterTokenizer *pRet;
void *pUserdata = 0;
const char *zBase = "unicode61";
fts5_tokenizer_v2 *pV2 = 0;
if( nArg>0 ){
zBase = azArg[0];
@@ -594,14 +593,15 @@ static int fts5PorterCreate(
pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
if( pRet ){
memset(pRet, 0, sizeof(PorterTokenizer));
rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
int nArg2 = (nArg>0 ? nArg-1 : 0);
const char **azArg2 = (nArg2 ? &azArg[1] : 0);
rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
const char **az2 = (nArg2 ? &azArg[1] : 0);
memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
}
if( rc!=SQLITE_OK ){
@@ -1252,6 +1252,7 @@ static int fts5PorterTokenize(
void *pCtx,
int flags,
const char *pText, int nText,
const char *pLoc, int nLoc,
int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
@@ -1259,8 +1260,8 @@ static int fts5PorterTokenize(
sCtx.xToken = xToken;
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer.xTokenize(
p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
return p->tokenizer_v2.xTokenize(
p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
);
}
@@ -1290,40 +1291,46 @@ static int fts5TriCreate(
Fts5Tokenizer **ppOut
){
int rc = SQLITE_OK;
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
TrigramTokenizer *pNew = 0;
UNUSED_PARAM(pUnused);
if( pNew==0 ){
rc = SQLITE_NOMEM;
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
pNew->bFold = 1;
pNew->iFoldParam = 0;
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
pNew->bFold = (zArg[0]=='0');
}
}else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
rc = SQLITE_ERROR;
}else{
pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
}
}else{
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
}
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
}
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
}
}
}
*ppOut = (Fts5Tokenizer*)pNew;
@@ -1428,6 +1435,16 @@ int sqlite3Fts5TokenizerPattern(
return FTS5_PATTERN_NONE;
}
/*
** Return true if the tokenizer described by p->azArg[] is the trigram
** tokenizer. This tokenizer needs to be loaded before xBestIndex is
** called for the first time in order to correctly handle LIKE/GLOB.
*/
int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
}
/*
** Register all built-in tokenizers with FTS5.
*/
@@ -1438,7 +1455,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
} aBuiltin[] = {
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
{ "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
{ "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
};
@@ -1453,6 +1469,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
0
);
}
if( rc==SQLITE_OK ){
fts5_tokenizer_v2 sPorter = {
2,
fts5PorterCreate,
fts5PorterDelete,
fts5PorterTokenize
};
rc = pApi->xCreateTokenizer_v2(pApi,
"porter",
(void*)pApi,
&sPorter,
0
);
}
return rc;
}

3
ext/fts5/fts5_unicode2.c
View File

@@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
default: return 1; }
break;
default:
return 1;
}
return 0;
}

32
ext/fts5/fts5_vocab.c
View File

@@ -64,6 +64,7 @@ struct Fts5VocabCursor {
int nLeTerm; /* Size of zLeTerm in bytes */
char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
int colUsed; /* Copy of sqlite3_index_info.colUsed */
/* These are used by 'col' tables only */
int iCol;
@@ -90,9 +91,11 @@ struct Fts5VocabCursor {
/*
** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
*/
#define FTS5_VOCAB_TERM_EQ 0x01
#define FTS5_VOCAB_TERM_GE 0x02
#define FTS5_VOCAB_TERM_LE 0x04
#define FTS5_VOCAB_TERM_EQ 0x0100
#define FTS5_VOCAB_TERM_GE 0x0200
#define FTS5_VOCAB_TERM_LE 0x0400
#define FTS5_VOCAB_COLUSED_MASK 0xFF
/*
@@ -269,11 +272,13 @@ static int fts5VocabBestIndexMethod(
int iTermEq = -1;
int iTermGe = -1;
int iTermLe = -1;
int idxNum = 0;
int idxNum = (int)pInfo->colUsed;
int nArg = 0;
UNUSED_PARAM(pUnused);
assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
for(i=0; i<pInfo->nConstraint; i++){
struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
if( p->usable==0 ) continue;
@@ -365,7 +370,7 @@ static int fts5VocabOpenMethod(
if( rc==SQLITE_OK ){
pVTab->zErrMsg = sqlite3_mprintf(
"no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
);
);
rc = SQLITE_ERROR;
}
}else{
@@ -525,9 +530,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
switch( pTab->eType ){
case FTS5_VOCAB_ROW:
if( eDetail==FTS5_DETAIL_FULL ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
/* Do not bother counting the number of instances if the "cnt"
** column is not being read (according to colUsed). */
if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
while( iPos<nPos ){
u32 ii;
fts5FastGetVarint32(pPos, iPos, ii);
if( ii==1 ){
/* New column in the position list */
fts5FastGetVarint32(pPos, iPos, ii);
}else{
/* An instance - increment pCsr->aCnt[] */
pCsr->aCnt[0]++;
}
}
}
pCsr->aDoc[0]++;
@@ -625,6 +640,7 @@ static int fts5VocabFilterMethod(
if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);
if( pEq ){
zTerm = (const char *)sqlite3_value_text(pEq);

23
ext/fts5/test/fts5_common.tcl
View File

@@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} {
sort_poslist $res
}
proc fts5_test_insttoken {cmd iInst iToken} {
$cmd xInstToken $iInst $iToken
}
proc fts5_test_collist {cmd} {
set res [list]
@@ -78,6 +82,9 @@ proc fts5_test_columnsize {cmd} {
proc fts5_columntext {cmd iCol} {
$cmd xColumnText $iCol
}
proc fts5_columnlocale {cmd iCol} {
$cmd xColumnLocale $iCol
}
proc fts5_test_columntext {cmd} {
set res [list]
@@ -87,6 +94,14 @@ proc fts5_test_columntext {cmd} {
set res
}
proc fts5_test_columnlocale {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnLocale $i]
}
set res
}
proc fts5_test_columntotalsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -114,6 +129,10 @@ proc fts5_test_rowcount {cmd} {
$cmd xRowCount
}
proc fts5_test_rowid {cmd} {
$cmd xRowid
}
proc test_queryphrase_cb {cnt cmd} {
upvar $cnt L
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -161,17 +180,21 @@ proc fts5_aux_test_functions {db} {
foreach f {
fts5_test_columnsize
fts5_test_columntext
fts5_test_columnlocale
fts5_test_columntotalsize
fts5_test_poslist
fts5_test_poslist2
fts5_test_collist
fts5_test_insttoken
fts5_test_tokenize
fts5_test_rowcount
fts5_test_rowid
fts5_test_all
fts5_test_queryphrase
fts5_test_phrasecount
fts5_columntext
fts5_columnlocale
fts5_queryphrase
fts5_collist
} {

4
ext/fts5/test/fts5aa.test
View File

@@ -440,7 +440,7 @@ do_execsql_test 16.1 {
proc funk {} {
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
set fd [db incrblob main n1_data block 10]
fconfigure $fd -encoding binary -translation binary
fconfigure $fd -translation binary
# puts -nonewline $fd "\x44\x45"
close $fd
}
@@ -453,7 +453,7 @@ db func funk funk
# statement no longer fails.
#
do_catchsql_test 16.2 {
SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {0 {{} -1e-06 {}}}
# {1 {SQL logic error}}

2
ext/fts5/test/fts5ab.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ab
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ac.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ac
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ad.test
View File

@@ -18,7 +18,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ad
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ae.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ae
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5af.test
View File

@@ -18,7 +18,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5af
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ag.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ag
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

13
ext/fts5/test/fts5ah.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ah
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -163,6 +163,17 @@ do_execsql_test 1.8.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
} {10000}
do_execsql_test 1.8.3 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text';
} {4999}
do_execsql_test 1.8.4 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text';
} {0}
do_catchsql_test 1.9 {
SELECT * FROM t1('*xy');
} {1 {unknown special query: xy}}
} ;# foreach_detail_mode
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

2
ext/fts5/test/fts5ai.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ai
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5aj.test
View File

@@ -19,7 +19,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aj
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ak.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ak
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

12
ext/fts5/test/fts5al.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5al
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}
# Check that the second and subsequent rank= constraints are ignored.
#
do_catchsql_test 4.3.3 {
SELECT *, rank FROM t3
WHERE t3 MATCH 'a' AND
rank MATCH 'nosuch()' AND
rank MATCH 'rowidmod(3)'
ORDER BY rank ASC
} {1 {unable to use function MATCH in the requested context}}
} ;# foreach_detail_mode

2
ext/fts5/test/fts5alter.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5alter
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5auto.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5auto
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

26
ext/fts5/test/fts5aux.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -377,4 +377,28 @@ do_catchsql_test 12.3.3 {
SELECT fts5_collist(t1, 1) FROM t1('one AND two');
} {0 1}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
INSERT INTO t1 VALUES('a b c'), ('d e f');
PRAGMA integrity_check;
} {ok}
do_catchsql_test 13.2 {
SELECT highlight(t1, 0, '[', ']') FROM t1
} {0 {{a b c} {d e f}}}
do_execsql_test 13.3 {
PRAGMA writable_schema = 1;
UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
WHERE name = 't1';
}
db close
sqlite3 db test.db
do_catchsql_test 13.4 {
SELECT highlight(t1, 0, '[', ']') FROM t1
} {1 {SQL logic error}}
finish_test

71
ext/fts5/test/fts5aux2.test Normal file
View File

@@ -0,0 +1,71 @@
# 2024 June 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the auxiliary function APIs.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
INSERT INTO x1 VALUES('a b', 'c d');
INSERT INTO x1 VALUES('d e', 'a b');
INSERT INTO x1 VALUES('a b', 'e f');
INSERT INTO x1 VALUES('d e', 'c d');
}
fts5_aux_test_functions db
do_execsql_test 1.1 {
SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
} [list [list {*}{
columnsize {2 2}
columntext {{d e} {a b}}
columntotalsize {8 8}
poslist {}
tokenize {{d e} {a b}}
rowcount 4
}]]
do_execsql_test 1.2 {
SELECT fts5_test_columntext(x1) FROM x1
} {
{{a b} {c d}}
{{d e} {a b}}
{{a b} {e f}}
{{d e} {c d}}
}
do_execsql_test 1.3 {
SELECT fts5_test_rowid(x1) FROM x1
} {
1 2 3 4
}
do_execsql_test 1.4 {
SELECT fts5_test_phrasecount(x1) FROM x1
} {
0 0 0 0
}
do_catchsql_test 1.5 {
SELECT fts5_queryphrase(x1, 0) FROM x1
} {1 SQLITE_RANGE}
do_execsql_test 1.6 {
SELECT fts5_test_rowcount(x1) FROM x1
} {4 4 4 4}
finish_test

2
ext/fts5/test/fts5auxdata.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5auxdata
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5bigpl.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5bigpl
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

166
ext/fts5/test/fts5blob.test Normal file
View File

@@ -0,0 +1,166 @@
# 2024 July 30
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file verifies that:
#
# * blob values may be written to locale=0 tables.
#
# * blob values - other than fts5_locale() values - may not be written
# to locale=0 tables. This is an SQLITE_MISMATCH error
#
# * blob values may be returned by queries on the external-content table
# of a locale=0 table.
#
# * blob values not may be returned by queries on the external-content
# table of a locale=1 table, apart from fts5_locale() blobs. This is an
# SQLITE_MISMATCH error.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5blob
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
# Test that blobs may be stored in normal locale=0 tables.
#
foreach {tn enc} {
1 utf8
2 utf16
} {
reset_db
fts5_aux_test_functions db
execsql "PRAGMA encoding = $enc"
execsql "
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
"
do_execsql_test 1.$tn.0 {
CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance');
INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546');
INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546');
INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz');
}
do_execsql_test 1.$tn.1 {
SELECT rowid, quote(x), quote(y) FROM t1
} {
1 555 X'0000000041424320444546'
2 666 X'41424300444546'
3 777 'xyz'
}
do_execsql_test 1.$tn.2 {
DELETE FROM t1 WHERE rowid=2;
DELETE FROM t1 WHERE rowid=1;
}
do_execsql_test 1.$tn.3 {
PRAGMA integrity_check;
} {ok}
}
#--------------------------------------------------------------------------
# Test that a blob may be stored and retrieved in an unindexed column of
# a regular table with locale=1.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
}
do_execsql_test 2.1 {
select rowid, x, quote(y) FROM t1
} {
12 twelve X'0000000041424320444546'
}
#--------------------------------------------------------------------------
# Test that blobs may not be written to any type of table with locale=1
# set. Except, they may be written to UNINDEXED columns.
#
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a, b);
CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1);
CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2);
CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=);
}
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
#--------------------------------------------------------------------------
# Test that fts5_locale() values may not be written to any type of table
# without locale=1 set. Even to an UNINDEXED column.
#
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a, b);
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2);
CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=);
CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED);
}
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b)
VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
do_catchsql_test 3.4 {
INSERT INTO x4(rowid, a, b, c)
VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc'));
} {1 {fts5_locale() requires locale=1}}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
}
foreach {tn sql} {
1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') }
2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') }
3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') }
} {
do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}}
}
finish_test

17
ext/fts5/test/fts5cat.test
View File

@@ -55,5 +55,22 @@ do_execsql_test 1.5 {
SELECT * FROM t4t
} {สนามกีฬา 1 1}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 "
CREATE VIRTUAL TABLE x1 USING fts5(c,
tokenize=\"unicode61 categories ' \t'\");
"
do_catchsql_test 2.1 "
CREATE VIRTUAL TABLE x2 USING fts5(c,
tokenize=\"unicode61 categories 'N*\t\tMYZ'\");
" {1 {error in tokenizer constructor}}
do_catchsql_test 2.2 "
CREATE VIRTUAL TABLE x2 USING fts5(c,
tokenize=\"unicode61 categories 'N*\t\tXYZ'\");
" {1 {error in tokenizer constructor}}
finish_test

2
ext/fts5/test/fts5colset.test
View File

@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 4.1 {
SELECT * FROM t1 WHERE rowid MATCH 'a'
} {1 {unable to use function MATCH in the requested context}}
} {1 {no query solution}}
}
#-------------------------------------------------------------------------

2
ext/fts5/test/fts5columnsize.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5columnsize
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5config.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5config
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

39
ext/fts5/test/fts5content.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5content
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -328,4 +328,41 @@ do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') }
do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one}
do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 9.1 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, 'one two three');
INSERT INTO t1 VALUES(2, 'one two three');
CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a);
INSERT INTO ft(ft) VALUES('rebuild');
}
do_execsql_test 9.2 {
SELECT rowid, b FROM ft('two');
} {
1 {one two three}
2 {one two three}
}
do_execsql_test 9.3 {
DELETE FROM t1 WHERE a=2;
}
do_catchsql_test 9.4 {
SELECT rowid FROM ft('two');
} {0 {1 2}}
do_catchsql_test 9.5 {
SELECT * FROM ft('two');
} {1 {fts5: missing row 2 from content table 'main'.'t1'}}
fts5_aux_test_functions db
do_catchsql_test 9.6 {
SELECT rowid, fts5_columntext(ft, 0) FROM ft('two');
} {1 SQLITE_CORRUPT_VTAB}
finish_test

22
ext/fts5/test/fts5contentless.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5contentless
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -267,4 +267,24 @@ do_execsql_test 8.2 {
SELECT rowid FROM ft('four');
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0);
INSERT INTO ft VALUES('hello world');
INSERT INTO ft VALUES('one two three');
}
do_catchsql_test 9.1 {
INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
} {0 {}}
do_catchsql_test 9.2 {
CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2);
} {1 {malformed contentless_delete=... directive}}
do_catchsql_test 9.3 {
CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11);
} {1 {malformed contentless_delete=... directive}}
finish_test

2
ext/fts5/test/fts5contentless2.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5contentless2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5contentless3.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5contentless3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5contentless4.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5contentless4
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

3
ext/fts5/test/fts5contentless5.test
View File

@@ -15,11 +15,12 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5contentless5
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
unset -nocomplain res
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1);

23
ext/fts5/test/fts5corrupt.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -99,6 +99,27 @@ sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 3.1 {
DELETE FROM t3_content WHERE rowid = 3;
SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}}
#--------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x);
INSERT INTO t2 VALUES('one two three');
INSERT INTO t2 VALUES('four five six');
INSERT INTO t2 VALUES('seven eight nine');
INSERT INTO t2 VALUES('ten eleven twelve');
}
do_execsql_test 4.1 {
SELECT hex(block) FROM t2_data WHERE id=1;
} {040C}
do_execsql_test 4.2 {
UPDATE t2_data SET block = X'0402' WHERE id=1
}
breakpoint
do_catchsql_test 4.3 {
DELETE FROM t2 WHERE rowid=3
} {1 {database disk image is malformed}}
finish_test

7
ext/fts5/test/fts5corrupt2.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -100,6 +100,7 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
set all [db eval {SELECT rowid FROM t1}]
sqlite3_db_config db DEFENSIVE 0
unset -nocomplain res
for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
do_execsql_test 2.$i.1 {
BEGIN;
@@ -152,7 +153,7 @@ foreach {tn hdr} {
execsql BEGIN
set fd [db incrblob main x3_data block $rowid]
fconfigure $fd -encoding binary -translation binary
fconfigure $fd -translation binary
set existing [read $fd [string length $hdr]]
seek $fd 0
puts -nonewline $fd $hdr
@@ -238,7 +239,7 @@ foreach {tn hdr} {
execsql BEGIN
set fd [db incrblob main x5_data block $rowid]
fconfigure $fd -encoding binary -translation binary
fconfigure $fd -translation binary
puts -nonewline $fd $hdr
close $fd

171
ext/fts5/test/fts5corrupt3.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -4267,7 +4267,7 @@ do_test 35.0 {
do_catchsql_test 35.1 {
SELECT * FROM t1 WHERE t1 MATCH 'e*';
} {1 {database disk image is malformed}}
} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}}
#-------------------------------------------------------------------------
reset_db
@@ -8958,7 +8958,6 @@ do_catchsql_test 61.2 {
SELECT * FROM t3 ORDER BY rowid;
} {/*malformed database schema*/}
breakpoint
#-------------------------------------------------------------------------
do_test 62.0 {
sqlite3 db {}
@@ -10768,6 +10767,7 @@ do_catchsql_test 73.1 {
reset_db
do_test 74.0 {
sqlite3 db {}
sqlite3_fts5_register_matchinfo db
db deserialize [decode_hexdb {
| size 106496 pagesize 4096 filename x.db
| page 1 offset 0
@@ -14587,14 +14587,19 @@ do_test 74.0 {
| end x.db
}]} {}
do_catchsql_test 74.1 {
SELECT rowid, quote(matchinfo(t1,'p<>xyb<s')) FROM t1 WHERE t1 MATCH 'e*';
do_catchsql_test 74.0.5 {
SELECT matchinfo(1,2);
} {1 {unable to use function matchinfo in the requested context}}
do_catchsql_test 74.1 {
SELECT rowid, quote(matchinfo(t1,'pxyb<s')) FROM t1 WHERE t1 MATCH 'e*';
} {1 {unrecognized matchinfo flag: <}}
#-------------------------------------------------------------------------
reset_db
do_test 75.0 {
sqlite3 db {}
sqlite3_fts5_register_matchinfo db
db deserialize [decode_hexdb {
| size 32768 pagesize 4096 filename crash-033d665d5caa8d.db
| page 1 offset 0
@@ -14791,7 +14796,7 @@ do_test 75.0 {
do_catchsql_test 75.1 {
SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*';
} {1 {unable to use function matchinfo in the requested context}}
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
reset_db
@@ -15519,6 +15524,160 @@ do_catchsql_test 80.1 {
SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank);
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
reset_db
do_test 81.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 40960 pagesize 4096 filename crash-44e8035a976422.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........
| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................
| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................
| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L
| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P
| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n..........
| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C.
| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C
| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA
| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts
| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro
| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable
| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU
| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN
| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1'
| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V.......
| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2
| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T
| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config'
| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY,
| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID
| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2
| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte
| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE
| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id
| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY
| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2)
| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2
| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT
| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx'
| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg
| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY(
| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI
| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU....
| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data
| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T
| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i
| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR
| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO
| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table
| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU
| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN
| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],.
| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c
| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V...
| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf
| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA
| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con
| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K
| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R
| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab
| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d
| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA
| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize'
| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM
| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB
| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet
| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont
| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE
| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id
| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY
| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2
| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet
| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA
| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx
| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p
| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY
| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W
| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU...
| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat
| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE
| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'(
| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA
| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL
| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl
| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT
| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI
| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin
| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize
| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.)
| page 2 offset 4096
| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............|
| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*..............
| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........
| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7.....
| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a.....
| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g...
| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i...
| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*...
| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%...
| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g...
| 3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42 .....h........jB
| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7.......
| 4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01 .t.....0a.....b.
| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g.....
| 4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04 ..h.......i.....
| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................
| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 3 offset 8192
| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................
| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................
| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................
| page 4 offset 12288
| 0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................
| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig
| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i
| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i......
| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i
| page 5 offset 16384
| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................
| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................
| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................
| page 6 offset 20480
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| page 7 offset 24576
| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................
| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A.
| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a
| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c....
| 4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g..
| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i......
| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................
| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 8 offset 28672
| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 9 offset 32768
| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................
| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................
| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig
| 4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69 h!i.......g h i
| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i......
| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i
| page 10 offset 36864
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| end crash-44e8035a976422.db
}]} {}
do_catchsql_test 81.2 {
UPDATE t1 SET b=zeroblob(299);
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test

2
ext/fts5/test/fts5corrupt4.test
View File

@@ -14,7 +14,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt4
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

231
ext/fts5/test/fts5corrupt5.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt5
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -1222,6 +1222,235 @@ do_catchsql_test 8.1 {
SELECT rowid FROM t1('a* NOT ý‘') ;
} {0 {1 2 3 4 5 6 7}}
#-------------------------------------------------------------------------
reset_db
do_test 9.0 {
sqlite3 db {}
db deserialize [decode_hexdb {
.open --hexdb
| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db
| page 1 offset 0
| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3.
| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........
| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................
| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6
| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............
| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet
| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE
| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta
| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c
| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB
| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k
| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v)
| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[.
| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d
| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize
| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't
| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN
| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE
| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...!
| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont
| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR
| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c
| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG
| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY,
| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i....
| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt
| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB
| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi
| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P
| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid
| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT
| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t
| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da
| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE
| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT
| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY
| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8..
| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR
| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB
| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5
| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c).........
| page 3 offset 8192
| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................
| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J..........
| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00.........
| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160
| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4.
| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5.....
| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000...
| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary..
| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................
| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp
| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d
| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat...........
| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e
| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable...........
| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................
| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................
| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................
| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension..
| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4...
| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5.......
| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc.........
| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........
| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........
| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load.........
| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max...........
| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory...........
| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n
| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase...........
| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................
| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit.........
| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree.........
| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............
| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................
| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................
| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................
| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V
| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab.....
| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x.........
| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................
| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................
| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................
| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................
| 3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01 ................
| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................
| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................
| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................
| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................
| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................
| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................
| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G..........
| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$.
| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............
| page 4 offset 12288
| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................
| page 5 offset 16384
| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t
| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./..........
| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$......
| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5....
| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$..
| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%.
| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI
| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA
| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE..
| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d ..%..THREADS1FE=
| 3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRdRIM.!..3..OM
| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO
| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O
| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI
| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3..
| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS
| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3..
| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000
| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3.
| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000
| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3
| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500
| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....%
| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB
| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB
| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE.
| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR
| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E
| 3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49 NABKE MEMSYS5XBI
| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL
| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE
| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME
| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....%
| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB
| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB
| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE.
| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO
| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E
| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI
| 3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c NARY....9..ENABL
| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE
| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE
| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....#
| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI
| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL
| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE...
| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X
| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB
| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY..
| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4
| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN
| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM.
| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY.
| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE.
| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS
| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM..
| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR
| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO
| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG
| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM
| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0
| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY'
| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67 ...C..COMPILER<g
| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060
| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C
| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4.
| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM
| page 6 offset 20480
| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$...........
| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................
| 32: 1f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.`
| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(.
| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................
| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#......
| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!......
| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . ..............
| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................
| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................
| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................
| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................
| 3920: 06 15 f3 00 12 02 01 01 06 15 03 00 12 02 01 01 ................
| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................
| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................
| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................
| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................
| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................
| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................
| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................
| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................
| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................
| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................
| page 7 offset 24576
| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................
| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version.
| page 8 offset 28672
| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................
| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr
| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb
| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 00 00 00 00 uild....opti....
| end crash-c76a16c24c8ba6.db
}]} {}
#.testctrl prng_seed 1 db
#.testctrl internal_functions
#.testctrl json_selfcheck on
#
do_execsql_test 9.1 {
UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
SAVEPOINT a;
UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
}
do_catchsql_test 9.2 {
DELETE FROM t1;
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test

2
ext/fts5/test/fts5corrupt6.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt6
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5corrupt7.test
View File

@@ -13,7 +13,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt7
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5dlidx.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5dlidx
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5doclist.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5doclist
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5ea.test
View File

@@ -16,7 +16,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ea
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

6
ext/fts5/test/fts5eb.test
View File

@@ -13,7 +13,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5eb
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -86,13 +86,13 @@ do_execsql_test 3.0 {
INSERT INTO e1 VALUES ('just a few words with a / inside');
}
do_execsql_test 3.1 {
SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
} {1 -1e-06}
do_execsql_test 3.2 {
SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
} 1
do_execsql_test 3.3 {
SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
} {1 -1e-06}
do_execsql_test 3.4 "

52
ext/fts5/test/fts5expr.test Normal file
View File

@@ -0,0 +1,52 @@
# 2024 August 8
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5expr
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a);
INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test');
}
do_execsql_test 1.1 {
SELECT rowid FROM x1('expr');
} {113}
for {set ii 0} {$ii < 300} {incr ii} {
set expr "expr "
append expr [string repeat "NOT abcd " $ii]
if {$ii<257} {
set res {0 113}
} else {
set res {1 {fts5 expression tree is too large (maximum depth 256)}}
}
do_catchsql_test 1.1.$ii {
SELECT rowid FROM x1($expr)
} $res
}
do_execsql_test 1.2 {
SELECT rowid FROM x1 WHERE a MATCH '"..."'
} {}
finish_test

2
ext/fts5/test/fts5fault6.test
View File

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault6
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

14
ext/fts5/test/fts5fault8.test
View File

@@ -57,7 +57,6 @@ foreach_detail_mode $testprefix {
} ;# foreach_detail_mode...
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x2 USING fts5(a);
INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
@@ -80,5 +79,18 @@ do_faultsim_test 4 -faults oom-* -prep {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}
set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}}
do_faultsim_test 5 -faults oom-t* -prep {
faultsim_restore_and_reopen
execsql { PRAGMA temp_store = memory }
} -body {
execsql { PRAGMA integrity_check }
} -test {
if {[string match {*error code=7*} $testresult]==0} {
faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR
}
}
finish_test

2
ext/fts5/test/fts5faultG.test
View File

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultG
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

13
ext/fts5/test/fts5faultH.test
View File

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultG
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -127,7 +127,7 @@ do_execsql_test 3.0 {
COMMIT;
}
do_faultsim_test 3 -faults oom* -prep {
do_faultsim_test 3.1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1('BBB AND AAA');
@@ -136,6 +136,15 @@ do_faultsim_test 3 -faults oom* -prep {
faultsim_integrity_check
faultsim_test_result {0 {10 35}}
}
do_faultsim_test 3.2 -faults oom* -prep {
} -body {
execsql {
SELECT count(*) FROM t1('BBB');
}
} -test {
faultsim_integrity_check
faultsim_test_result {0 27}
}
finish_test

261
ext/fts5/test/fts5faultI.test Normal file
View File

@@ -0,0 +1,261 @@
# 2010 June 15
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultI
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set ::testprefix fts5faultI
do_execsql_test 1.0 {
PRAGMA encoding = utf16;
CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1);
INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram');
}
faultsim_save_and_close
faultsim_restore_and_reopen
do_faultsim_test 1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM t1(fts5_locale('en_US', 'origintext'));
}
} -test {
faultsim_test_result {0 1}
}
do_faultsim_test 2 -faults oom* -prep {
faultsim_restore_and_reopen
execsql {
SELECT * FROM t1('ascii');
}
} -body {
execsql {
UPDATE t1 SET rowid=rowid+1;
}
} -test {
faultsim_test_result {0 {}}
}
fts5_aux_test_functions db
do_faultsim_test 3 -faults oom* -prep {
} -body {
execsql {
SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*');
}
} -test {
faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE w1 USING fts5(a);
}
faultsim_save_and_close
do_faultsim_test 4 -faults oom* -prep {
faultsim_restore_and_reopen
execsql {
BEGIN;
INSERT INTO w1 VALUES('token token token');
}
} -body {
execsql {
INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()');
}
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 5 -faults oom* -prep {
faultsim_restore_and_reopen
execsql {
BEGIN;
INSERT INTO w1 VALUES('one');
SAVEPOINT one;
INSERT INTO w1 VALUES('two');
ROLLBACK TO one;
}
} -body {
execsql {
INSERT INTO w1 VALUES('string');
}
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE w1 USING fts5(a);
INSERT INTO w1 VALUES('one two three');
}
fts5_aux_test_functions db
do_faultsim_test 5 -faults oom* -prep {
} -body {
execsql {
SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two');
}
} -test {
faultsim_test_result {0 two} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE w1 USING fts5(a);
INSERT INTO w1 VALUES('one two three');
}
fts5_aux_test_functions db
faultsim_save_and_close
do_faultsim_test 6 -faults oom* -prep {
faultsim_restore_and_reopen
db eval {
BEGIN;
INSERT INTO w1 VALUES('four five six');
SAVEPOINT abc;
INSERT INTO w1 VALUES('seven eight nine');
SAVEPOINT def;
INSERT INTO w1 VALUES('ten eleven twelve');
}
} -body {
execsql {
RELEASE abc;
}
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE w1 USING fts5(a);
INSERT INTO w1 VALUES('one two three');
INSERT INTO w1 VALUES('three two one');
DELETE FROM w1_content WHERE rowid=1;
}
faultsim_save_and_close
do_faultsim_test 7 -faults oom* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM w1 }
} -body {
execsql {
PRAGMA integrity_check;
}
} -test {
}
#-------------------------------------------------------------------------
reset_db
fts5_tclnum_register db
fts5_aux_test_functions db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize = "tclnum query", detail=columns
);
INSERT INTO ft VALUES('one two three i ii iii');
INSERT INTO ft VALUES('four five six iv v vi');
INSERT INTO ft VALUES('eight nine ten viii ix x');
} {}
do_faultsim_test 8.1 -faults oom* -prep {
} -body {
execsql {
SELECT fts5_test_collist (ft) FROM ft('one two');
}
} -test {
faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM}
}
do_faultsim_test 8.2 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM ft('one two') ORDER BY rank;
}
} -test {
faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE ft USING fts5(x);
INSERT INTO ft VALUES('one two three i ii iii');
INSERT INTO ft VALUES('four five six iv v vi');
INSERT INTO ft VALUES('eight nine ten viii ix x');
} {}
faultsim_save_and_close
do_faultsim_test 9.1 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
UPDATE ft SET rowid=4 WHERE rowid=1
}
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 9.2 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three'
}
} -test {
faultsim_test_result {0 1}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE ft USING fts5(x, locale=1);
INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii'));
INSERT INTO ft VALUES('four five six iv v vi');
INSERT INTO ft VALUES('eight nine ten viii ix x');
} {}
do_execsql_test 10.1 {
SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
} {hello}
faultsim_save_and_close
do_faultsim_test 10 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
}
} -test {
faultsim_test_result {0 hello}
}
finish_test

1
ext/fts5/test/fts5first.test
View File

@@ -22,6 +22,7 @@ do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
}
unset -nocomplain res
foreach {tn expr ok} {
1 {^abc} 1
2 {^abc + def} 1

2
ext/fts5/test/fts5full.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5full
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5hash.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5hash
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

56
ext/fts5/test/fts5integrity.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5integrity
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -153,6 +153,7 @@ do_execsql_test 5.3 {
INSERT INTO gg(gg) VALUES('integrity-check');
}
unset -nocomplain res
do_test 5.4.1 {
set ok 0
for {set i 0} {$i < 10000} {incr i} {
@@ -355,4 +356,57 @@ do_execsql_test 11.4 {
PRAGMA integrity_check(t2);
} {ok}
#-------------------------------------------------------------------
reset_db
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
INSERT INTO x1 VALUES('one', 'two');
INSERT INTO x1 VALUES('three', 'four');
INSERT INTO x1 VALUES('five', 'six');
}
do_execsql_test 12.2 {
PRAGMA integrity_check
} {ok}
db close
sqlite3 db test.db -readonly 1
explain_i {
PRAGMA integrity_check
}
do_execsql_test 12.3 {
PRAGMA integrity_check
} {ok}
#-------------------------------------------------------------------
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
INSERT INTO t1 VALUES('a b c'), ('d e f');
PRAGMA integrity_check;
} {ok}
db close
sqlite3 db test.db
do_catchsql_test 13.2 {
PRAGMA integrity_check;
} {0 ok}
do_execsql_test 13.3 {
PRAGMA writable_schema = 1;
UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
WHERE name = 't1';
}
db close
sqlite3 db test.db
breakpoint
do_catchsql_test 13.4 {
PRAGMA integrity_check;
} {1 {SQL logic error}}
finish_test

2
ext/fts5/test/fts5interrupt.test
View File

@@ -33,6 +33,7 @@ proc progress_handler {args} {
return 0
}
unset -nocomplain res
foreach {tn sql} {
1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') }
2 { COMMIT }
@@ -64,4 +65,3 @@ foreach {tn sql} {
}
finish_test

2
ext/fts5/test/fts5lastrowid.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5lastrowid
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

667
ext/fts5/test/fts5locale.test Normal file
View File

@@ -0,0 +1,667 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5locale
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc transform_token {locale token} {
switch -- $locale {
reverse {
set ret ""
foreach c [split $token ""] {
set ret "$c$ret"
}
set token $ret
}
default {
# no-op
}
}
set token
}
proc tcl_create {args} { return "tcl_tokenize" }
proc tcl_tokenize {tflags text} {
set iToken 1
set bSkip 0
if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 }
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
incr iToken
if {(($iToken) % ($bSkip + 1))} continue
set w [transform_token [sqlite3_fts5_locale] $w]
sqlite3_fts5_token $w $iStart $iEnd
}
}
#-------------------------------------------------------------------------
# Check that queries can have a locale attached to them.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
INSERT INTO t1 VALUES('abc');
INSERT INTO t1 VALUES('cba');
} {}
do_execsql_test 1.1 {
SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') );
} {1 abc}
do_execsql_test 1.2 {
SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
} {2 cba}
#-------------------------------------------------------------------------
# Test that the locale= option exists and seems to accept values. And
# that fts5_locale() values may only be inserted into an internal-content
# table if the locale=1 option was specified.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl);
CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl);
CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance);
}
do_catchsql_test 2.2.1 {
CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2);
} {1 {malformed locale=... directive}}
do_catchsql_test 2.2.2 {
CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111);
} {1 {malformed locale=... directive}}
do_catchsql_test 2.3 {
INSERT INTO b1(b1, rank) VALUES('locale', 0);
} {1 {SQL logic error}}
do_execsql_test 2.4 {
INSERT INTO b1 VALUES('abc', 'one two three');
INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
}
do_execsql_test 2.5 {
INSERT INTO b2 VALUES('abc', 'one two three');
}
do_catchsql_test 2.6 {
INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six'));
} {1 {fts5_locale() requires locale=1}}
do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1}
do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {}
do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2
do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2
do_execsql_test 2.11 {
SELECT x, quote(y) FROM b1
} {
abc {'one two three'}
def {'four five six'}
}
do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } {
{'four five six'}
}
do_execsql_test 2.13 {
INSERT INTO b1(b1) VALUES('integrity-check');
}
do_execsql_test 2.14 {
INSERT INTO b1(b1) VALUES('rebuild');
}
do_execsql_test 2.15 {
INSERT INTO b1(b1) VALUES('integrity-check');
}
do_execsql_test 2.16 {
DELETE FROM b1 WHERE rowid=2
}
do_execsql_test 2.17 {
INSERT INTO b1(b1) VALUES('integrity-check');
}
do_execsql_test 2.18 {
INSERT INTO b1(rowid, x, y) VALUES(
test_setsubtype(45, 76), 'abc def', 'def abc'
);
INSERT INTO b1(b1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
# Test the 'delete' command with contentless tables.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1);
CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance);
INSERT INTO c1 VALUES('hello world');
INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') );
}
do_execsql_test 3.2 {
SELECT DISTINCT term FROM c2 ORDER BY 1
} {
eerht eno hello owt world
}
do_execsql_test 3.3 {
INSERT INTO c1(c1, rowid, x)
VALUES('delete', 2, fts5_locale('reverse', 'one two three') );
}
do_execsql_test 3.4 {
SELECT DISTINCT term FROM c2 ORDER BY 1
} {
hello world
}
#-------------------------------------------------------------------------
# Test that an UPDATE that updates a subset of the columns does not
# magically discard the locale from those columns not updated.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 4.1 {
CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl);
CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance);
INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def');
INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello'));
}
do_execsql_test 4.2 {
SELECT DISTINCT term FROM d2 ORDER BY 1
} {
abc def ghi olleh
}
do_execsql_test 4.3 {
UPDATE d1 SET x='jkl' WHERE rowid=2;
}
do_execsql_test 4.4 {
SELECT DISTINCT term FROM d2 ORDER BY 1
} {
abc def jkl olleh
}
do_execsql_test 4.5 {
SELECT rowid, * FROM d1
} {
1 abc def
2 jkl hello
}
do_execsql_test 4.6 {
UPDATE d1 SET rowid=4 WHERE rowid=2
}
do_execsql_test 4.7 {
SELECT rowid, * FROM d1
} {
1 abc def
4 jkl hello
}
fts5_aux_test_functions db
do_execsql_test 4.8.1 {
SELECT fts5_test_columntext(d1) FROM d1('jkl')
} {{jkl hello}}
do_execsql_test 4.8.2 {
SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello'))
} {{jkl hello}}
do_execsql_test 4.9 {
SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello'))
} {{{} reverse}}
do_execsql_test 4.10 {
SELECT fts5_test_columnlocale(d1) FROM d1
} {
{{} {}}
{{} reverse}
}
#-------------------------------------------------------------------------
# Test that if an fts5_locale() value is written to an UNINDEXED
# column it is stored as text. This is so that blobs and other values
# can also be stored as is.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE t1 USING fts5(
x, y UNINDEXED, locale=1, tokenize=tcl
);
INSERT INTO t1(rowid, x, y) VALUES(111,
fts5_locale('reverse', 'one two three'),
fts5_locale('reverse', 'four five six')
);
}
do_execsql_test 5.2 {
SELECT rowid, x, y FROM t1
} {
111 {one two three} {four five six}
}
do_execsql_test 5.3 {
SELECT typeof(c0), typeof(c1) FROM t1_content
} {
blob text
}
#-------------------------------------------------------------------------
foreach {tn opt} {
1 {}
2 {, columnsize=0}
} {
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 6.$tn.1 "
CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt);
"
do_execsql_test 6.$tn.2 {
INSERT INTO y1(rowid, t) VALUES
(1, fts5_locale('second', 'the city of London')),
(2, fts5_locale('second', 'shall have all the old')),
(3, fts5_locale('second', 'Liberties and Customs')),
(4, fts5_locale('second', 'which it hath been used to have'));
}
fts5_aux_test_functions db
do_execsql_test 5.$tn.3 {
SELECT fts5_test_columnsize(y1) FROM y1
} {
2 3 2 4
}
do_execsql_test 5.$tn.4 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.5 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.6 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
} {
4 4
}
do_execsql_test 5.$tn.7 {
SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
} {
4 {which it hath been used to [have]}
}
do_execsql_test 5.$tn.8 {
SELECT rowid,
highlight(y1, 0, '[', ']'),
snippet(y1, 0, '[', ']', '...', 10)
FROM y1('Liberties + Customs');
} {
3 {[Liberties and Customs]}
{[Liberties and Customs]}
}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
}
do_catchsql_test 6.1 {
INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world'));
} {1 {fts5_locale() requires locale=1}}
do_execsql_test 6.2 {
SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') );
} {text text}
#--------------------------------------------------------------------------
# Test that fts5_locale() works with external-content tables.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 7.1 {
CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT);
CREATE VIEW v1 AS
SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1;
CREATE VIRTUAL TABLE ft USING fts5(
bb, tt, locale=1, tokenize=tcl, content=v1
);
INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL);
INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse');
INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second');
}
do_execsql_test 7.2 {
INSERT INTO ft(ft) VALUES('rebuild');
INSERT INTO ft(ft) VALUES('integrity-check');
}
do_execsql_test 7.3 {
SELECT rowid, quote(bb), quote(tt) FROM ft
} {
1 NULL {'one two three'}
2 '7800616263' {'four five six'}
3 '000000007800616263' {'seven eight nine'}
}
do_execsql_test 7.4 { SELECT rowid FROM ft('six'); }
do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2
fts5_aux_test_functions db
do_execsql_test 7.6 {
SELECT fts5_test_columnlocale(ft) FROM ft;
} {
{{} {}} {{} reverse} {{} second}
}
#-------------------------------------------------------------------------
# Test that the porter tokenizer works with locales.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl");
CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
INSERT INTO ft(rowid, tt) VALUES
(111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')),
(222, fts5_locale('reverse', 'This value may also be set'));
}
do_execsql_test 8.1 {
SELECT DISTINCT term FROM vocab ORDER BY 1
} {
a eb eulav osla sihT te the token yam
}
#-------------------------------------------------------------------------
# Test that position-lists (used by xInst, xPhraseFirst etc.) work with
# locales and modes other than detail=full.
#
foreach {tn detail} {
1 detail=full
2 detail=none
3 detail=column
} {
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
do_execsql_test 9.$tn.0 "
CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail);
"
do_execsql_test 9.$tn.1 {
CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
INSERT INTO ft(rowid, tt) VALUES
(-1, fts5_locale('second', 'it is an ancient mariner'));
}
do_execsql_test 9.$tn.2 {
SELECT DISTINCT term FROM vocab
} {an it mariner}
do_execsql_test 9.$tn.3 {
SELECT highlight(ft, 0, '[', ']') FROM ft('mariner')
} {{it is an ancient [mariner]}}
}
#-------------------------------------------------------------------------
# Check some corrupt fts5_locale() blob formats are detected.
#
foreach_detail_mode $::testprefix {
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
fts5_aux_test_functions db
do_execsql_test 10.1 {
CREATE TABLE x1(ii INTEGER PRIMARY KEY, x);
CREATE VIRTUAL TABLE ft USING fts5(x,
content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0
);
CREATE VIRTUAL TABLE ft2 USING fts5(
x, locale=1, detail=%DETAIL%, columnsize=0
);
}
foreach {tn v} {
1 X'001122'
2 X'0011223344'
3 X'00E0B2EB68656c6c6f'
4 X'00E0B2EB0068656c6c6f'
} {
do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') }
do_execsql_test 10.2.$tn.1 { DELETE FROM x1; }
do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) "
do_catchsql_test 10.2.$tn.3 {
INSERT INTO ft(ft) VALUES('rebuild');
} {1 {SQL logic error}}
do_catchsql_test 10.2.$tn.4 "
SELECT * FROM ft( test_setsubtype($v, 76) );
" {1 {SQL logic error}}
do_execsql_test 10.2.$tn.5 {
INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
}
if {"%DETAIL%"!="full"} {
do_catchsql_test 10.2.$tn.6 {
SELECT fts5_test_poslist(ft) FROM ft('world');
} {1 SQLITE_ERROR}
do_catchsql_test 10.2.$tn.7 {
SELECT fts5_test_columnsize(ft) FROM ft('world');
} {1 SQLITE_ERROR}
do_catchsql_test 10.2.$tn.7 {
SELECT fts5_test_columnlocale(ft) FROM ft('world');
} {1 SQLITE_ERROR}
}
do_catchsql_test 10.2.$tn.8 {
SELECT * FROM ft('hello')
} {1 {SQL logic error}}
do_catchsql_test 10.2.$tn.9 {
PRAGMA integrity_check;
} {0 ok}
do_execsql_test 10.2.$tn.10 {
DELETE FROM x1;
INSERT INTO x1(ii, x) VALUES(1, 'hello world');
}
do_catchsql_test 10.2.$tn.11 "
INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
" {1 {SQL logic error}}
do_catchsql_test 10.2.$tn.12 "
INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
" {1 {SQL logic error}}
do_execsql_test 10.2.$tn.13 {
INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
}
do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v"
do_catchsql_test 10.2.$tn.15 {
PRAGMA integrity_check;
} {1 {SQL logic error}}
do_execsql_test 10.2.$tn.16 {
DELETE FROM ft2_content;
INSERT INTO ft2(ft2) VALUES('rebuild');
}
}
}
#-------------------------------------------------------------------------
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
fts5_aux_test_functions db
do_execsql_test 11.0 {
CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1);
INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three'));
}
do_catchsql_test 11.1 {
SELECT fts5_columnlocale(x1, -1) FROM x1('two');
} {1 SQLITE_RANGE}
do_catchsql_test 11.2 {
SELECT fts5_columnlocale(x1, 1) FROM x1('two');
} {1 SQLITE_RANGE}
#-------------------------------------------------------------------------
#
reset_db
do_test 12.0 {
list [catch {
sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create
} msg] $msg
} {1 {error in fts5_api.xCreateTokenizer_v2()}}
#-------------------------------------------------------------------------
# Tests for auxiliary function fts5_get_locale().
#
reset_db
# Check that if the table does not support locale=1, fts5_get_locale()
# always returns NULL.
do_execsql_test 13.1.0 {
CREATE VIRTUAL TABLE nolocale USING fts5(a, b);
INSERT INTO nolocale VALUES('one two three', 'four five six');
INSERT INTO nolocale VALUES('three two one', 'seven eight nine');
}
do_execsql_test 13.1.1 {
SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale;
} {1 1}
do_execsql_test 13.1.2 {
SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two');
} {1}
do_execsql_test 13.1.3 {
SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two');
} {1 1}
do_execsql_test 13.1.4 {
SELECT
fts5_get_locale(nolocale, 1) IS NULL
FROM nolocale('three AND two') ORDER BY rank
} {1 1}
do_catchsql_test 13.1.5 {
SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two');
} {1 {column index out of range}}
do_catchsql_test 13.1.6 {
SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two');
} {1 {column index out of range}}
do_catchsql_test 13.1.7 {
SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two');
} {1 {wrong number of arguments to function fts5_get_locale()}}
do_catchsql_test 13.1.8 {
SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two');
} {1 {wrong number of arguments to function fts5_get_locale()}}
do_catchsql_test 13.1.9 {
SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two');
} {1 {non-integer argument passed to function fts5_get_locale()}}
# Check that if the table does support locale=1, fts5_get_locale()
# returns the locale of the identified row/column.
do_execsql_test 13.2.0 {
CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1);
INSERT INTO ft VALUES(
fts5_locale('th_TH', 'one two three'), 'four five six seven'
);
INSERT INTO ft VALUES(
'three two one', fts5_locale('en_AU', 'seven eight nine')
);
}
do_execsql_test 13.2.1 {
SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft
} { 'th_TH' NULL NULL 'en_AU' }
do_execsql_test 13.2.2 {
SELECT
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three')
} { 'th_TH' NULL NULL 'en_AU' }
do_execsql_test 13.2.3 {
SELECT
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three') ORDER BY rank
} { NULL 'en_AU' 'th_TH' NULL }
do_execsql_test 13.2.4 {
SELECT
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three') ORDER BY rowid
} { 'th_TH' NULL NULL 'en_AU' }
do_execsql_test 13.2.5 {
SELECT
quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three') ORDER BY rowid
} { 'th_TH' NULL NULL 'en_AU' }
do_catchsql_test 13.2.6 {
SELECT
quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three') ORDER BY rowid
} {1 {non-integer argument passed to function fts5_get_locale()}}
do_catchsql_test 13.2.7 {
SELECT
quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1))
FROM ft('one AND three') ORDER BY rowid
} {1 {non-integer argument passed to function fts5_get_locale()}}
finish_test

2
ext/fts5/test/fts5matchinfo.test
View File

@@ -517,7 +517,7 @@ fts5_aux_test_functions db
do_execsql_test 15.3 {
SELECT fts5_test_all(t1) FROM t1 LIMIT 1;
} {
{columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
{columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
}
finish_test

2
ext/fts5/test/fts5merge.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5merge
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

138
ext/fts5/test/fts5misc.test
View File

@@ -35,21 +35,21 @@ do_catchsql_test 1.1.2 {
do_catchsql_test 1.2.1 {
SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id');
} {0 {{}}}
} {1 {no such cursor: 4}}
do_catchsql_test 1.2.2 {
SELECT a FROM t1
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'));
} {0 {}}
} {1 {no such cursor: 6}}
do_catchsql_test 1.3.1 {
SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads');
} {1 {no such cursor: 2}}
} {1 {no such cursor: 1}}
do_catchsql_test 1.3.2 {
SELECT a FROM t1
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
} {1 {no such cursor: 2}}
} {1 {no such cursor: 1}}
db close
sqlite3 db test.db
@@ -59,6 +59,11 @@ do_catchsql_test 1.3.3 {
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
} {1 {no such cursor: 1}}
fts5_aux_test_functions db
do_catchsql_test 1.3.4 {
SELECT fts5_columntext(t1) FROM t1('*reads');
} {1 {no such cursor: 1}}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
@@ -535,5 +540,130 @@ do_execsql_test 19.0 {
COMMIT;
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a);
INSERT INTO x1(rowid, a) VALUES
(1, 'a b c d'),
(2, 'x b c d'),
(3, 'x y z d'),
(4, 'a y c x');
}
do_execsql_test 20.1 {
SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b';
} {1}
do_execsql_test 20.2 {
SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y';
} {4}
do_execsql_test 20.3 {
SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y';
} {1 4 3}
do_execsql_test 20.4 {
SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd');
} {1 4 3}
do_execsql_test 20.5 {
SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd');
} {3 1}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 21.0 {
CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT);
CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y);
INSERT INTO t1 VALUES(1, 'one', 'i');
INSERT INTO t1 VALUES(2, 'two', 'ii');
INSERT INTO t1 VALUES(3, 'tree', 'iii');
INSERT INTO xyz(xyz) VALUES('rebuild');
}
do_execsql_test 21.1 {
UPDATE xyz SET y='TWO' WHERE rowid=2;
UPDATE t1 SET y='TWO' WHERE ii=2;
}
do_execsql_test 21.2 {
PRAGMA integrity_check
} {ok}
breakpoint
sqlite3_db_config db DEFENSIVE 1
do_execsql_test 21.3 {
CREATE TABLE xyz_notashadow(x, y);
DROP TABLE xyz_notashadow;
}
sqlite3_db_config db DEFENSIVE 0
#-------------------------------------------------------------------------
reset_db
do_execsql_test 22.0 {
SELECT fts5(NULL);
} {{}}
do_execsql_test 22.1 {
SELECT count(*) FROM (
SELECT fts5_source_id()
)
} {1}
execsql_pp {
SELECT fts5_source_id()
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 23.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1 VALUES('one + two + three');
INSERT INTO x1 VALUES('one + xyz + three');
INSERT INTO x1 VALUES('xyz + two + xyz');
}
do_execsql_test 23.1 {
SELECT rowid FROM x1('one + two + three');
} {1}
do_execsql_test 23.2 {
SELECT rowid FROM x1('^".." AND one');
} {}
do_execsql_test 23.3 {
SELECT rowid FROM x1('abc NEAR ".." NEAR def');
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 24.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
INSERT INTO t1(a) VALUES('a');
}
do_execsql_test 24.2 {
SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1;
} {-1e-06}
do_execsql_test 24.3 {
SELECT rank FROM ( SELECT rank FROM t1('a NOT <20> NOT def') ) ORDER BY 1;
} {-1e-06}
do_execsql_test 24.4 {
SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') );
} {-1e-06}
#-------------------------------------------------------------------------
reset_db
fts5_aux_test_functions db
do_execsql_test 25.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content='');
INSERT INTO t1(a) VALUES('a b c');
}
do_execsql_test 25.0 {
SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {{}}
finish_test

2
ext/fts5/test/fts5near.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5near
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5optimize.test
View File

@@ -14,7 +14,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5optimize
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5optimize2.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5optimize2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5optimize3.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5optimize2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

22
ext/fts5/test/fts5origintext.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -261,8 +261,8 @@ do_execsql_test 5.3 {
reset_db
sqlite3_fts5_register_origintext db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, y, tokenize='origintext unicode61', detail=%DETAIL%
CREATE VIRTUAL TABLE ft USING fts5(
x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
);
INSERT INTO ft VALUES('One Two', 'Three two');
@@ -291,6 +291,22 @@ do_execsql_test 6.3 {
SELECT rowid, tokens(ft) FROM ft('Three*');
} {1 {{}} 2 {{}}}
fts5_aux_test_functions db
do_catchsql_test 6.4 {
SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
} {1 SQLITE_RANGE}
do_catchsql_test 6.5 {
SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
} {1 SQLITE_RANGE}
do_catchsql_test 6.6 {
CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
} {1 {malformed tokendata=... directive}}
do_catchsql_test 6.7 {
CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
} {1 {malformed tokendata=... directive}}
}
finish_test

2
ext/fts5/test/fts5origintext2.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5origintext3.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5origintext4.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext4
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5origintext5.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5origintext
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5phrase.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5phrase
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5plan.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5plan
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5porter.test
View File

@@ -17,7 +17,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5porter
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5porter2.test
View File

@@ -18,7 +18,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5porter2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5prefix.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5prefix
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5prefix2.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5prefix2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5query.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5query
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5rank.test
View File

@@ -15,7 +15,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rank
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

2
ext/fts5/test/fts5rebuild.test
View File

@@ -14,7 +14,7 @@
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rebuild
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return

Some files were not shown because too many files have changed in this diff Show More