database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Alternative implementation of fts5 locale=1 feature that allows blobs to be stored in indexed columns of fts5 locale=1 tables.

Browse Source 
FossilOrigin-Name: 55c5c119a0a77fac2c9f46d718ef78c0f33ed3520e10c240cf5bf1801e0586ee
This commit is contained in:
dan
2024-09-10 16:19:31 +00:00
parent cd889c7a88
commit 7a41b48cf9
10 changed files with 408 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ext/fts5/fts5Int.h
View File

@ -636,18 +636,13 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
int sqlite3Fts5FlushToDisk(Fts5Table*);
int sqlite3Fts5ExtractText(
Fts5Config *pConfig,
sqlite3_value *pVal, /* Value to extract text from */
int bContent, /* Loaded from content table */
int *pbResetTokenizer, /* OUT: True if ClearLocale() required */
const char **ppText, /* OUT: Pointer to text buffer */
int *pnText /* OUT: Size of (*ppText) in bytes */
);
void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
);
/*
** End of interface to code in fts5.c.

9
ext/fts5/fts5_config.c
View File

@ -516,6 +516,15 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){
}
}
}
if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
for(i=0; i<p->nCol; i++){
if( p->abUnindexed[i]==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
}else{
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
}
}
}
assert( p->zContentExprlist==0 );
p->zContentExprlist = (char*)buf.p;

349
ext/fts5/fts5_main.c
View File

@ -1259,7 +1259,7 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
** valid until after the final call to sqlite3Fts5Tokenize() that will use
** the locale.
*/
static void fts5SetLocale(
static void sqlite3Fts5SetLocale(
Fts5Config *pConfig,
const char *zLocale,
int nLocale
@ -1270,11 +1270,10 @@ static void fts5SetLocale(
}
/*
** Clear any locale configured by an earlier call to fts5SetLocale() or
** sqlite3Fts5ExtractText().
** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
*/
void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
fts5SetLocale(pConfig, 0, 0);
sqlite3Fts5SetLocale(pConfig, 0, 0);
}
/*
@ -1302,118 +1301,33 @@ int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
}
/*
** This function is used to extract utf-8 text from an sqlite3_value. This
** is usually done in order to tokenize it. For example, when:
**
** * a value is written to an fts5 table,
** * a value is deleted from an FTS5_CONTENT_NORMAL table,
** * a value containing a query expression is passed to xFilter()
**
** and so on.
**
** This function handles 2 cases:
**
** 1) Ordinary values. The text can be extracted from these using
** sqlite3_value_text().
**
** 2) Combination text/locale blobs created by fts5_locale(). There
** are several cases for these:
**
** * Blobs that have the 16-byte header, and
** * Blobs read from the content table of a locale=1 regular
** content table.
**
** The first case above has the 16 byte FTS5_LOCALE_HDR(pConfig)
** header. It is an error if a blob read from the content table of
** an external content table does not have the required header. A blob
** read from the content table of a regular locale=1 table does not
** have the header. This is to save space.
**
** If successful, SQLITE_OK is returned and output parameters (*ppText)
** and (*pnText) are set to point to a buffer containing the extracted utf-8
** text and its length in bytes, respectively. The buffer is not
** nul-terminated. It has the same lifetime as the sqlite3_value object
** from which it is extracted.
**
** Parameter bContent must be true if the value was read from an indexed
** column (i.e. not UNINDEXED) of the on disk content.
**
** If pbResetTokenizer is not NULL and if case (2) is used, then
** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls
** use the locale. In this case (*pbResetTokenizer) is set to true before
** returning, to indicate that the caller must call sqlite3Fts5ClearLocale()
** to clear the locale after tokenizing the text.
** Value pVal is guaranteed to be an fts5_locale() value.
*/
int sqlite3Fts5ExtractText(
Fts5Config *pConfig,
sqlite3_value *pVal, /* Value to extract text from */
int bContent, /* True if indexed table content */
int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */
const char **ppText, /* OUT: Pointer to text buffer */
int *pnText /* OUT: Size of (*ppText) in bytes */
int sqlite3Fts5DecodeLocaleValue(
sqlite3_value *pVal,
const char **ppText,
int *pnText,
const char **ppLoc,
int *pnLoc
){
const char *pText = 0;
int nText = 0;
int rc = SQLITE_OK;
const char *p = sqlite3_value_blob(pVal);
int n = sqlite3_value_bytes(pVal);
int nLoc = 0;
/* 0: Do not decode blob
** 1: Decode blob, expect fts5_locale() header
** 2: Decode blob, expect no fts5_locale() header
*/
int eDecodeBlob = 0;
assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
assert( n>FTS5_LOCALE_HDR_SIZE );
assert( pbResetTokenizer==0 || *pbResetTokenizer==0 );
assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE );
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
if( bContent
&& pConfig->bLocale
&& pConfig->eContent==FTS5_CONTENT_NORMAL
){
eDecodeBlob = 2;
}else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
eDecodeBlob = 1;
}else if( bContent && pConfig->bLocale ){
return SQLITE_ERROR;
for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){
if( nLoc==(n-1) ){
return SQLITE_MISMATCH;
}
}
*ppLoc = &p[FTS5_LOCALE_HDR_SIZE];
*pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE;
if( eDecodeBlob ){
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
int nLocale = 0;
/* Unless this blob was read from the %_content table of an
** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale()
** header. Check for this. If it is not found, return an error. */
if( eDecodeBlob==1 ){
pBlob += FTS5_LOCALE_HDR_SIZE;
nBlob -= FTS5_LOCALE_HDR_SIZE;
}
for(nLocale=0; nLocale<nBlob; nLocale++){
if( pBlob[nLocale]==0x00 ) break;
}
if( nLocale==nBlob || nLocale==0 ){
rc = SQLITE_ERROR;
}else{
pText = (const char*)&pBlob[nLocale+1];
nText = nBlob-nLocale-1;
if( pbResetTokenizer ){
fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
*pbResetTokenizer = 1;
}
}
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
*ppText = pText;
*pnText = nText;
return rc;
*ppText = &p[nLoc+1];
*pnText = n - nLoc - 1;
return SQLITE_OK;
}
/*
@ -1422,8 +1336,8 @@ int sqlite3Fts5ExtractText(
** the text of the expression, and sets output variable (*pzText) to
** point to a nul-terminated buffer containing the expression.
**
** If pVal was an fts5_locale() value, then fts5SetLocale() is called to
** set the tokenizer to use the specified locale.
** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called
** to set the tokenizer to use the specified locale.
**
** If output variable (*pbFreeAndReset) is set to true, then the caller
** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer
@ -1435,24 +1349,22 @@ static int fts5ExtractExprText(
char **pzText, /* OUT: nul-terminated buffer of text */
int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */
){
const char *zText = 0;
int nText = 0;
int rc = SQLITE_OK;
int bReset = 0;
*pbFreeAndReset = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText);
if( rc==SQLITE_OK ){
if( bReset ){
*pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
if( rc!=SQLITE_OK ){
sqlite3Fts5ClearLocale(pConfig);
}else{
*pbFreeAndReset = 1;
}
}else{
*pzText = (char*)zText;
if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
*pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
}
*pbFreeAndReset = 1;
}else{
*pzText = (char*)sqlite3_value_text(pVal);
*pbFreeAndReset = 0;
}
return rc;
@ -1992,22 +1904,16 @@ static int fts5UpdateMethod(
else{
int eType1 = sqlite3_value_numeric_type(apVal[1]);
int ii;
for(ii=0; ii<pConfig->nCol; ii++){
sqlite3_value *pVal = apVal[ii+2];
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
int isLocale = sqlite3Fts5IsLocaleValue(pConfig, pVal);
if( pConfig->bLocale ){
if( isLocale==0 && pConfig->abUnindexed[ii]==0 ){
rc = SQLITE_MISMATCH;
goto update_out;
}
}else{
if( isLocale ){
fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
rc = SQLITE_MISMATCH;
goto update_out;
}
/* It is an error to write an fts5_locale() value to a table without
** the locale=1 option. */
if( pConfig->bLocale==0 ){
int ii;
for(ii=0; ii<pConfig->nCol; ii++){
sqlite3_value *pVal = apVal[ii+2];
if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
rc = SQLITE_MISMATCH;
goto update_out;
}
}
}
@ -2166,11 +2072,11 @@ static int fts5ApiTokenize_v2(
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
int rc = SQLITE_OK;
fts5SetLocale(pTab->pConfig, pLoc, nLoc);
sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pTab->pConfig,
FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
);
fts5SetLocale(pTab->pConfig, 0, 0);
sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
return rc;
}
@ -2198,6 +2104,38 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}
/*
** Argument pStmt is an SQL statement of the type used by Fts5Cursor.
*/
static int fts5TextFromStmt(
Fts5Config *pConfig,
sqlite3_stmt *pStmt,
int iCol,
const char **ppText,
int *pnText
){
sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1);
const char *pLoc = 0;
int nLoc = 0;
int rc = SQLITE_OK;
if( pConfig->bLocale
&& pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
}else{
*ppText = (const char*)sqlite3_value_text(pVal);
*pnText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){
pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol);
nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol);
}
}
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
return rc;
}
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
@ -2217,10 +2155,8 @@ static int fts5ApiColumnText(
}else{
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
Fts5Config *pConfig = pTab->pConfig;
int bContent = (pConfig->abUnindexed[iCol]==0);
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn);
rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
sqlite3Fts5ClearLocale(pTab->pConfig);
}
}
return rc;
@ -2262,17 +2198,15 @@ static int fts5CsrPoslist(
rc = fts5SeekCursor(pCsr, 0);
}
for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
const char *z = 0;
int n = 0;
int bReset = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprPopulatePoslists(
pConfig, pCsr->pExpr, aPopulator, i, z, n
);
}
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
sqlite3_free(aPopulator);
@ -2458,17 +2392,14 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
if( pConfig->abUnindexed[i]==0 ){
const char *z = 0;
int n = 0;
int bReset = 0;
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
pCsr->aColumnSize[i] = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX,
z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
sqlite3Fts5ClearLocale(pConfig);
}
}
}
@ -2740,37 +2671,14 @@ static int fts5ApiColumnLocale(
){
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
/* Load the value into pVal. pVal is a locale/text pair iff:
**
** 1) It is an SQLITE_BLOB, and
** 2) Either the FTS5_LOCALE_HDR header is present, or else the
** value was loaded from an FTS5_CONTENT_NORMAL table.
**
** If condition (1) is met but condition (2) is not, it is an error.
*/
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
if( sqlite3Fts5IsLocaleValue(pConfig, pVal)==0 ){
rc = SQLITE_ERROR;
}
pBlob += FTS5_LOCALE_HDR_SIZE;
nBlob -= FTS5_LOCALE_HDR_SIZE;
}
if( rc==SQLITE_OK ){
int nLocale = 0;
for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
if( nLocale==nBlob || nLocale==0 ){
rc = SQLITE_ERROR;
}else{
/* A locale/text pair */
*pzLocale = (const char*)pBlob;
*pnLocale = nLocale;
}
}
const char *zDummy = 0;
int nDummy = 0;
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
if( rc==SQLITE_OK ){
*pzLocale = pConfig->t.pLocale;
*pnLocale = pConfig->t.nLocale;
}
sqlite3Fts5ClearLocale(pConfig);
}
}
@ -2991,58 +2899,6 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
return rc;
}
/*
** Value pVal was read from column iCol of the FTS5 table. This function
** returns it to the owner of pCtx via a call to an sqlite3_result_xxx()
** function. This function deals with the same cases as
** sqlite3Fts5ExtractText():
**
** 1) Ordinary values. These can be returned using sqlite3_result_value().
**
** 2) Blobs from fts5_locale(). The text is extracted from these and
** returned via sqlite3_result_text(). The locale is discarded.
*/
static void fts5ExtractValueFromColumn(
sqlite3_context *pCtx,
Fts5Config *pConfig,
int iCol,
sqlite3_value *pVal
){
assert( pConfig->eContent!=FTS5_CONTENT_NONE );
if( pConfig->bLocale
&& sqlite3_value_type(pVal)==SQLITE_BLOB
&& pConfig->abUnindexed[iCol]==0
){
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
int ii;
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
if( nBlob<FTS5_LOCALE_HDR_SIZE
|| memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE)
){
sqlite3_result_error_code(pCtx, SQLITE_ERROR);
return;
}else{
pBlob += FTS5_LOCALE_HDR_SIZE;
nBlob -= FTS5_LOCALE_HDR_SIZE;
}
}
for(ii=0; ii<nBlob && pBlob[ii]; ii++);
if( ii==0 || ii==nBlob ){
sqlite3_result_error_code(pCtx, SQLITE_ERROR);
}else{
const char *pText = (const char*)&pBlob[ii+1];
sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT);
}
return;
}
sqlite3_result_value(pCtx, pVal);
}
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
@ -3099,8 +2955,21 @@ static int fts5ColumnMethod(
rc = fts5SeekCursor(pCsr, 1);
if( rc==SQLITE_OK ){
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal);
if( pConfig->bLocale
&& pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
const char *z = 0;
int n = 0;
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT);
}
}else{
sqlite3_result_value(pCtx, pVal);
}
}
pConfig->pzErrmsg = 0;
}
}

243
ext/fts5/fts5_storage.c
View File

@ -73,6 +73,30 @@ struct Fts5Storage {
#define FTS5_STMT_REPLACE_CONFIG 10
#define FTS5_STMT_SCAN 11
/*
** Return a pointer to a buffer obtained from sqlite3_malloc() that contains
** nBind comma-separated question marks. e.g. if nBind is passed 5, this
** function returns "?,?,?,?,?".
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op and
** NULL is returned immediately. Or, if the attempt to malloc a buffer
** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise,
** if it is SQLITE_OK when this function is called and the malloc() succeeds,
** *pRc is left unchanged.
*/
static char *fts5BindingsList(int *pRc, int nBind){
char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2);
if( zBind ){
int ii;
for(ii=0; ii<nBind; ii++){
zBind[ii*2] = '?';
zBind[ii*2 + 1] = ',';
}
zBind[ii*2-1] = '\0';
}
return zBind;
}
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
@ -141,19 +165,20 @@ static int fts5StorageGetStmt(
);
break;
case FTS5_STMT_INSERT_CONTENT:
case FTS5_STMT_REPLACE_CONTENT: {
int nCol = pC->nCol + 1;
case FTS5_STMT_INSERT_CONTENT: {
int nCol = 0;
char *zBind;
int i;
zBind = sqlite3_malloc64(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
nCol = 1 + pC->nCol;
if( pC->bLocale ){
for(i=0; i<pC->nCol; i++){
if( pC->abUnindexed[i]==0 ) nCol++;
}
zBind[i*2-1] = '\0';
}
zBind = fts5BindingsList(&rc, nCol);
if( zBind ){
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
sqlite3_free(zBind);
}
@ -344,7 +369,7 @@ int sqlite3Fts5StorageOpen(
if( bCreate ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
int nDefn = 32 + pConfig->nCol*10;
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
@ -356,6 +381,14 @@ int sqlite3Fts5StorageOpen(
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
if( pConfig->bLocale ){
for(i=0; i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i]==0 ){
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
}
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
sqlite3_free(zDefn);
@ -507,7 +540,8 @@ static int fts5StorageDeleteFromIndex(
sqlite3_value *pVal = 0;
const char *pText = 0;
int nText = 0;
int bReset = 0;
const char *pLoc = 0;
int nLoc = 0;
assert( pSeek==0 || apVal==0 );
assert( pSeek!=0 || apVal!=0 );
@ -517,10 +551,19 @@ static int fts5StorageDeleteFromIndex(
pVal = apVal[iCol-1];
}
rc = sqlite3Fts5ExtractText(
pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
);
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale && pSeek ){
pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol);
nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
pText, nText, (void*)&ctx, fts5StorageInsertCallback
@ -529,7 +572,7 @@ static int fts5StorageDeleteFromIndex(
if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
}
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
}
@ -788,20 +831,35 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
int nLoc = 0; /* Size of pLoc in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@ -884,33 +942,46 @@ int sqlite3Fts5StorageContentInsert(
}else{
sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
int i; /* Counter variable */
int nIndexed = 0; /* Number indexed columns seen */
rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
if( pInsert ) sqlite3_clear_bindings(pInsert);
/* Bind the rowid value */
sqlite3_bind_value(pInsert, 1, apVal[1]);
/* Loop through values for user-defined columns. i=2 is the leftmost
** user-defined column. As is column 1 of pSavedRow. */
for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
int bUnindexed = pConfig->abUnindexed[i-2];
sqlite3_value *pVal = apVal[i];
nIndexed += !bUnindexed;
if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
/* This is an UPDATE statement, and column (i-2) was not modified.
** Retrieve the value from Fts5Storage.pSavedRow instead. */
pVal = sqlite3_column_value(p->pSavedRow, i-1);
}else if( sqlite3_value_type(pVal)==SQLITE_BLOB && pConfig->bLocale ){
assert( pConfig->bLocale );
assert( i>1 );
if( pConfig->abUnindexed[i-2] ){
if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
/* At attempt to insert an fts5_locale() value into an UNINDEXED
** column. Strip the locale away and just bind the text. */
const char *pText = 0;
int nText = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
continue;
}
}else{
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
assert( nBlob>4 );
sqlite3_bind_blob(pInsert, i, pBlob+16, nBlob-16, SQLITE_TRANSIENT);
continue;
if( pConfig->bLocale && bUnindexed==0 ){
sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed,
sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
);
}
}else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
const char *pText = 0;
const char *pLoc = 0;
int nText = 0;
int nLoc = 0;
assert( pConfig->bLocale );
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
if( bUnindexed==0 ){
int iLoc = pConfig->nCol + 1 + nIndexed;
sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
}
}
continue;
}
rc = sqlite3_bind_value(pInsert, i, pVal);
@ -948,23 +1019,37 @@ int sqlite3Fts5StorageIndexInsert(
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
int nLoc = 0; /* Size of pText in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = apVal[ctx.iCol+2];
int bDisk = 0;
if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
bDisk = 1;
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
}
}else{
pVal = apVal[ctx.iCol+2];
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
if( rc==SQLITE_OK ){
assert( bReset==0 || pConfig->bLocale );
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@ -1129,37 +1214,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
if( pConfig->abUnindexed[i]==0 ){
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(
pVal, &pText, &nText, &pLoc, &nLoc
);
}else{
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = i + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
ctx.iCol = i;
ctx.szCol = 0;
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
rc = sqlite3Fts5ExtractText(pConfig,
sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
/* If this is not a columnsize=0 database, check that the number
** of tokens in the value matches the aColSize[] value read from
** the %_docsize table. */
if( rc==SQLITE_OK
&& pConfig->bColumnsize
&& ctx.szCol!=aColSize[i]
){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);

6
ext/fts5/test/fts5blob.test
View File

@ -103,13 +103,13 @@ do_execsql_test 3.0 {
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
#--------------------------------------------------------------------------

5
ext/fts5/test/fts5faultI.test
View File

@ -256,15 +256,16 @@ do_faultsim_test 10.1 -faults oom* -prep {
faultsim_test_result {0 hello}
}
breakpoint
faultsim_save_and_close
do_faultsim_test 10.2 -faults oom* -prep {
do_faultsim_test 10.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
INSERT INTO ft VALUES(zeroblob(10000));
}
} -test {
faultsim_test_result {1 {datatype mismatch}}
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------

115
ext/fts5/test/fts5locale.test
View File

@ -73,6 +73,7 @@ do_execsql_test 1.2 {
SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
} {2 cba}
#-------------------------------------------------------------------------
# Test that the locale= option exists and seems to accept values. And
# that fts5_locale() values may only be inserted into an internal-content
@ -99,8 +100,11 @@ do_catchsql_test 2.3 {
INSERT INTO b1(b1, rank) VALUES('locale', 0);
} {1 {SQL logic error}}
do_execsql_test 2.4 {
do_execsql_test 2.4.1 {
INSERT INTO b1 VALUES('abc', 'one two three');
}
do_execsql_test 2.4.2 {
INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
}
@ -131,6 +135,7 @@ do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } {
do_execsql_test 2.13 {
INSERT INTO b1(b1) VALUES('integrity-check');
}
do_execsql_test 2.14 {
INSERT INTO b1(b1) VALUES('rebuild');
}
@ -149,7 +154,6 @@ do_execsql_test 2.18 {
INSERT INTO b1(rowid, x, y) VALUES(
test_setsubtype(45, 76), 'abc def', 'def abc'
);
INSERT INTO b1(b1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
@ -278,9 +282,9 @@ do_execsql_test 5.2 {
}
do_execsql_test 5.3 {
SELECT typeof(c0), typeof(c1) FROM t1_content
SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content
} {
blob text
text text text
}
#-------------------------------------------------------------------------
@ -305,37 +309,37 @@ foreach {tn opt} {
fts5_aux_test_functions db
do_execsql_test 5.$tn.3 {
do_execsql_test 6.$tn.3 {
SELECT fts5_test_columnsize(y1) FROM y1
} {
2 3 2 4
}
do_execsql_test 5.$tn.4 {
do_execsql_test 6.$tn.4 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.5 {
do_execsql_test 6.$tn.5 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.6 {
do_execsql_test 6.$tn.6 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
} {
4 4
}
do_execsql_test 5.$tn.7 {
do_execsql_test 6.$tn.7 {
SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
} {
4 {which it hath been used to [have]}
}
do_execsql_test 5.$tn.8 {
do_execsql_test 6.$tn.8 {
SELECT rowid,
highlight(y1, 0, '[', ']'),
snippet(y1, 0, '[', ']', '...', 10)
@ -473,7 +477,7 @@ foreach_detail_mode $::testprefix {
}
foreach {tn v} {
1 X'001122'
1 X'001152'
2 X'0011223344'
3 X'00E0B2EB68656c6c6f'
4 X'00E0B2EB0068656c6c6f'
@ -484,7 +488,7 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.3 {
INSERT INTO ft(ft) VALUES('rebuild');
} {1 {SQL logic error}}
} {0 {}}
do_catchsql_test 10.2.$tn.4 "
SELECT * FROM ft( test_setsubtype($v, 76) );
@ -494,23 +498,23 @@ foreach_detail_mode $::testprefix {
INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
}
if {"%DETAIL%"!="full"} {
do_catchsql_test 10.2.$tn.6 {
if {"%DETAIL%"=="full"} {
do_execsql_test 10.2.$tn.6 {
SELECT fts5_test_poslist(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {0.0.1}
do_catchsql_test 10.2.$tn.7 {
do_execsql_test 10.2.$tn.7.1 {
SELECT fts5_test_columnsize(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {1}
do_catchsql_test 10.2.$tn.7 {
do_execsql_test 10.2.$tn.7.2 {
SELECT fts5_test_columnlocale(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {{{}}}
}
do_catchsql_test 10.2.$tn.8 {
SELECT * FROM ft('hello')
} {1 {SQL logic error}}
SELECT count(*) FROM ft('hello')
} {0 1}
do_catchsql_test 10.2.$tn.9 {
PRAGMA integrity_check;
@ -527,7 +531,7 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.12 "
INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
" {1 {datatype mismatch}}
" {0 {}}
do_execsql_test 10.2.$tn.13 {
INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
@ -536,7 +540,7 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.15 {
PRAGMA integrity_check;
} {1 {SQL logic error}}
} {0 {{malformed inverted index for FTS5 table main.ft2}}}
do_execsql_test 10.2.$tn.16 {
DELETE FROM ft2_content;
@ -679,13 +683,66 @@ do_execsql_test 14.3 {
UPDATE ft SET b = fts5_locale('en_AU', 'world');
}
do_catchsql_test 14.4 {
INSERT INTO ft VALUES(X'abcd', X'1234');
} {1 {datatype mismatch}}
do_execsql_test 14.4 {
SELECT * FROM ft
} {hello world}
INSERT INTO ft VALUES(X'abcd', X'1234');
} {}
do_execsql_test 14.5 {
SELECT quote(a), quote(b) FROM ft
} {'hello' 'world' X'ABCD' X'1234'}
do_execsql_test 14.6 {
DELETE FROM ft;
INSERT INTO ft VALUES(NULL, 'null');
INSERT INTO ft VALUES(123, 'int');
INSERT INTO ft VALUES(345.0, 'real');
INSERT INTO ft VALUES('abc', 'text');
INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text');
SELECT a, typeof(a), b FROM ft
} {
{} null null
123 integer int
345.0 real real
abc text text
def text text
}
do_execsql_test 14.7 {
SELECT quote(c0), typeof(c0) FROM ft_content
} {
NULL null
123 integer
345.0 real
'abc' text
'def' text
}
#-------------------------------------------------------------------------
# Check that inserting UNINDEXED columns between indexed columns of a
# locale=1 table does not cause a problem.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
fts5_aux_test_functions db
do_execsql_test 15.1 {
CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl);
}
do_execsql_test 15.2 {
INSERT INTO ft VALUES('one', 'two', 'three');
INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three'));
}
do_execsql_test 15.3 {
SELECT c2, l2 FROM ft_content
} {three {} three loc}
do_execsql_test 15.4 {
SELECT c, fts5_columnlocale(ft, 2) FROM ft
} {three {} three loc}
finish_test

1
ext/fts5/test/fts5simple.test
View File

@ -371,6 +371,7 @@ foreach_detail_mode $testprefix {
CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
INSERT INTO x3 VALUES('a b c d e f');
}
breakpoint
do_execsql_test 16.1 {
SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
} {2.0.2}