1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00

Add a %_config table to fts5.

FossilOrigin-Name: 83491c56661ca78f96020ba68184bb3fb19e674f
This commit is contained in:
dan
2014-11-27 20:03:45 +00:00
parent cb62aae034
commit cb11e73fc2
17 changed files with 287 additions and 128 deletions

View File

@ -340,6 +340,11 @@ static int fts5InitVtab(
rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
}
/* Load the contents of %_config */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ConfigLoad(pConfig);
}
if( rc!=SQLITE_OK ){
fts5FreeVtab(pTab, 0);
pTab = 0;
@ -887,7 +892,8 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
** INSERT INTO fts(fts) VALUES($pVal)
** INSERT INTO fts(fts) VALUES($pCmd)
** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
**
** Argument pVal is the value assigned to column "fts" by the INSERT
** statement. This function returns SQLITE_OK if successful, or an SQLite
@ -897,28 +903,25 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){
** INSERT Directives" section of the documentation. It should be updated if
** more commands are added to this function.
*/
static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
const char *z = (const char*)sqlite3_value_text(pVal);
int n = sqlite3_value_bytes(pVal);
int rc = SQLITE_ERROR;
static int fts5SpecialCommand(
Fts5Table *pTab, /* Fts5 table object */
sqlite3_value *pCmd, /* Value inserted into special column */
sqlite3_value *pVal /* Value inserted into rowid column */
){
const char *z = (const char*)sqlite3_value_text(pCmd);
int rc = SQLITE_OK;
int bError = 0;
if( 0==sqlite3_stricmp("integrity-check", z) ){
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
}else
if( n>5 && 0==sqlite3_strnicmp("pgsz=", z, 5) ){
int pgsz = atoi(&z[5]);
if( pgsz<32 ) pgsz = 32;
sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz);
rc = SQLITE_OK;
}else
if( n>10 && 0==sqlite3_strnicmp("automerge=", z, 10) ){
int nAutomerge = atoi(&z[10]);
sqlite3Fts5IndexAutomerge(pTab->pIndex, nAutomerge);
rc = SQLITE_OK;
}else{
rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
if( rc==SQLITE_OK && bError ){
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
}
}
return rc;
}
@ -953,7 +956,9 @@ static int fts5UpdateMethod(
assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
return fts5SpecialCommand(pTab,
apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
);
}
eType0 = sqlite3_value_type(apVal[0]);
@ -1104,7 +1109,9 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
int *aInst;
int iBest = -1;
for(i=0; i<nIter; i++){
if( aIter[i].bEof==0 && (iBest<0 || aIter[i].iPos<iBest) ){
if( (aIter[i].bEof==0)
&& (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
){
iBest = i;
}
}

View File

@ -58,6 +58,8 @@ typedef struct Fts5Config Fts5Config;
/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
**
** And all information loaded from the %_config table.
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
@ -69,6 +71,10 @@ struct Fts5Config {
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
/* Values loaded from the %_config table */
int iCookie; /* Incremented when %_config is modified */
int pgsz; /* Approximate page size used in %_data */
};
int sqlite3Fts5ConfigParse(
@ -87,6 +93,12 @@ int sqlite3Fts5Tokenize(
void sqlite3Fts5Dequote(char *z);
/* Load the contents of the %_config table */
int sqlite3Fts5ConfigLoad(Fts5Config*);
/* Set the value of a single config attribute */
int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
@ -286,13 +298,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
*/
int sqlite3Fts5IndexInit(sqlite3*);
/*
** Set the page size to use when writing. It doesn't matter if this
** changes mid-transaction, or if inconsistent values are used by
** multiple clients.
*/
void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);
/*
@ -364,7 +369,7 @@ int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);
int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, char **pzErr);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
@ -381,6 +386,8 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);
int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/

View File

@ -21,6 +21,7 @@ typedef struct HighlightContext HighlightContext;
struct HighlightContext {
const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
Fts5Context *pFts; /* First arg to pass to pApi functions */
int nInst; /* Total number of phrase instances */
int iInst; /* Current phrase instance index */
int iStart; /* First token of current phrase */
int iEnd; /* Last token of current phrase */
@ -35,57 +36,61 @@ struct HighlightContext {
char *zOut; /* Output value */
};
static int fts5HighlightAppend(HighlightContext *p, const char *z, int n){
if( n<0 ) n = strlen(z);
p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
if( p->zOut==0 ) return SQLITE_NOMEM;
return SQLITE_OK;
/*
** Append text to the HighlightContext output string - p->zOut. Argument
** z points to a buffer containing n bytes of text to append. If n is
** negative, everything up until the first '\0' is appended to the output.
*/
static void fts5HighlightAppend(
int *pRc,
HighlightContext *p,
const char *z, int n
){
if( *pRc==SQLITE_OK ){
if( n<0 ) n = strlen(z);
p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
}
}
static int fts5HighlightCb(
void *pContext, /* Pointer to HighlightContext object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iStartOff, /* Start offset of token */
int iEndOff, /* End offset of token */
int iPos /* Position offset of token */
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
if( iPos==p->iStart ){
rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iStart - p->iOff);
p->iOff = iStart;
if( rc==SQLITE_OK ){
rc = fts5HighlightAppend(p, p->zOpen, -1);
}
}
if( rc==SQLITE_OK ){
rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iEnd - p->iOff);
p->iOff = iEnd;
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->iOff = iStartOff;
}
if( rc==SQLITE_OK && iPos==p->iEnd ){
if( iPos==p->iEnd ){
int bClose = 1;
do{
for(p->iInst++; rc==SQLITE_OK && p->iInst<p->nInst; p->iInst++){
int iP, iPCol, iOff;
rc = p->pApi->xInst(p->pFts, ++p->iInst, &iP, &iPCol, &iOff);
if( rc==SQLITE_RANGE || iPCol!=p->iCol ){
p->iStart = -1;
p->iEnd = -1;
rc = SQLITE_OK;
rc = p->pApi->xInst(p->pFts, p->iInst, &iP, &iPCol, &iOff);
if( iPCol!=p->iCol ){
p->iStart = p->iEnd = -1;
}else{
iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
int iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
if( iEnd<=p->iEnd ) continue;
if( iOff<=p->iEnd ) bClose = 0;
p->iStart = iOff;
p->iEnd = iEnd;
}
}while( 0 );
break;
}
if( rc==SQLITE_OK && bClose ){
rc = fts5HighlightAppend(p, p->zClose, -1);
if( bClose ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->iOff = iEndOff;
}
}
@ -107,34 +112,33 @@ static void fts5HighlightFunction(
sqlite3_result_error(pCtx, zErr, -1);
return;
}
memset(&ctx, 0, sizeof(HighlightContext));
ctx.iCol = sqlite3_value_int(apVal[0]);
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
ctx.pApi = pApi;
ctx.pFts = pFts;
rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
if( rc==SQLITE_OK ) rc = pApi->xInstCount(pFts, &ctx.nInst);
/* Find the first phrase instance in the right column. */
ctx.iStart = -1;
ctx.iEnd = -1;
while( rc==SQLITE_OK ){
for( ; ctx.iInst<ctx.nInst && rc==SQLITE_OK; ctx.iInst++){
int iP, iPCol, iOff;
rc = pApi->xInst(pFts, ctx.iInst, &iP, &iPCol, &iOff);
if( rc==SQLITE_OK && iPCol==ctx.iCol ){
if( iPCol==ctx.iCol ){
ctx.iStart = iOff;
ctx.iEnd = iOff - 1 + pApi->xPhraseSize(pFts, iP);
break;
}
ctx.iInst++;
}
if( rc==SQLITE_OK || rc==SQLITE_RANGE ){
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
}
if( rc==SQLITE_OK ){
rc = fts5HighlightAppend(&ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
}
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);

View File

@ -15,6 +15,8 @@
#include "fts5Int.h"
#define FTS5_DEFAULT_PAGE_SIZE 1000
/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters. The conversion is done in-place. If the
@ -295,4 +297,65 @@ int sqlite3Fts5Tokenize(
return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
}
int sqlite3Fts5ConfigSetValue(
Fts5Config *pConfig,
const char *zKey,
sqlite3_value *pVal,
int *pbBadkey
){
int rc = SQLITE_OK;
if( 0==sqlite3_stricmp(zKey, "cookie") ){
pConfig->iCookie = sqlite3_value_int(pVal);
}
else if( 0==sqlite3_stricmp(zKey, "pgsz") ){
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
pConfig->pgsz = sqlite3_value_int(pVal);
}else{
if( pbBadkey ) *pbBadkey = 1;
}
}
else if( 0==sqlite3_stricmp(zKey, "automerge") ){
// todo
}
else if( 0==sqlite3_stricmp(zKey, "rank") ){
// todo
}else{
if( pbBadkey ) *pbBadkey = 1;
}
return rc;
}
/*
** Load the contents of the %_config table into memory.
*/
int sqlite3Fts5ConfigLoad(Fts5Config *pConfig){
const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
char *zSql;
sqlite3_stmt *p = 0;
int rc;
/* Set default values */
pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
pConfig->iCookie = 0;
zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
sqlite3_free(zSql);
}
assert( rc==SQLITE_OK || p==0 );
if( rc==SQLITE_OK ){
while( SQLITE_ROW==sqlite3_step(p) ){
const char *zK = (const char*)sqlite3_column_text(p, 0);
sqlite3_value *pVal = sqlite3_column_value(p, 1);
sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
}
rc = sqlite3_finalize(p);
}
return rc;
}

View File

@ -41,8 +41,6 @@
**
*/
#define FTS5_DEFAULT_PAGE_SIZE 1000
#define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */
#define FTS5_MIN_MERGE 4 /* Minimum number of segments to merge */
#define FTS5_CRISIS_MERGE 16 /* Maximum number of segments to merge */
@ -290,7 +288,6 @@ typedef struct Fts5StructureSegment Fts5StructureSegment;
struct Fts5Index {
Fts5Config *pConfig; /* Virtual table configuration */
char *zDataTbl; /* Name of %_data table */
int pgsz; /* Target page size for this index */
int nMinMerge; /* Minimum input segments in a merge */
int nCrisisMerge; /* Maximum allowed segments per level */
int nWorkUnit; /* Leaf pages in a "unit" of work */
@ -2535,12 +2532,15 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
** Discard all data currently cached in the hash-tables.
*/
static void fts5IndexDiscardData(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
int i;
for(i=0; i<=pConfig->nPrefix; i++){
sqlite3Fts5HashClear(p->apHash[i]);
assert( p->apHash || p->nPendingData==0 );
if( p->apHash ){
Fts5Config *pConfig = p->pConfig;
int i;
for(i=0; i<=pConfig->nPrefix; i++){
sqlite3Fts5HashClear(p->apHash[i]);
}
p->nPendingData = 0;
}
p->nPendingData = 0;
}
/*
@ -2630,7 +2630,7 @@ static void fts5WriteBtreeTerm(
fts5WriteBtreeNEmpty(p, pWriter);
if( pPage->buf.n>=p->pgsz ){
if( pPage->buf.n>=p->pConfig->pgsz ){
/* pPage will be written to disk. The term will be written into the
** parent of pPage. */
i64 iRowid = FTS5_SEGMENT_ROWID(
@ -2761,7 +2761,7 @@ static void fts5WriteAppendTerm(
pWriter->bFirstRowidInDoclist = 1;
/* If the current leaf page is full, flush it to disk. */
if( pPage->buf.n>=p->pgsz ){
if( pPage->buf.n>=p->pConfig->pgsz ){
fts5WriteFlushLeaf(p, pWriter);
pWriter->bFirstRowidInPage = 1;
}
@ -2796,7 +2796,7 @@ static void fts5WriteAppendRowid(
pWriter->bFirstRowidInDoclist = 0;
pWriter->bFirstRowidInPage = 0;
if( pPage->buf.n>=p->pgsz ){
if( pPage->buf.n>=p->pConfig->pgsz ){
fts5WriteFlushLeaf(p, pWriter);
pWriter->bFirstRowidInPage = 1;
}
@ -2809,7 +2809,7 @@ static void fts5WriteAppendPoslistInt(
){
Fts5PageWriter *pPage = &pWriter->aWriter[0];
fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
if( pPage->buf.n>=p->pgsz ){
if( pPage->buf.n>=p->pConfig->pgsz ){
fts5WriteFlushLeaf(p, pWriter);
pWriter->bFirstRowidInPage = 1;
}
@ -2825,8 +2825,8 @@ static void fts5WriteAppendPoslistData(
const u8 *a = aData;
int n = nData;
while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pgsz ){
int nReq = p->pgsz - pPage->buf.n;
while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){
int nReq = p->pConfig->pgsz - pPage->buf.n;
int nCopy = 0;
while( nCopy<nReq ){
i64 dummy;
@ -3371,7 +3371,6 @@ int sqlite3Fts5IndexOpen(
memset(p, 0, sizeof(Fts5Index));
p->pConfig = pConfig;
p->pgsz = 1000;
p->nMinMerge = FTS5_MIN_MERGE;
p->nCrisisMerge = FTS5_CRISIS_MERGE;
p->nWorkUnit = FTS5_WORK_UNIT;
@ -3383,7 +3382,7 @@ int sqlite3Fts5IndexOpen(
int i;
Fts5Structure s;
rc = sqlite3Fts5CreateTable(
pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", pzErr
pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
);
if( rc==SQLITE_OK ){
memset(&s, 0, sizeof(Fts5Structure));
@ -3986,13 +3985,6 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
return rc;
}
/*
** Set the target page size for the index object.
*/
void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz){
p->pgsz = pgsz;
}
/*
** Set the minimum number of segments that an auto-merge operation should
** attempt to merge together. A value of 1 sets the object to use the

View File

@ -20,7 +20,7 @@ struct Fts5Storage {
int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
i64 nTotalRow; /* Total number of rows in FTS table */
i64 *aTotalSize; /* Total sizes of each column */
sqlite3_stmt *aStmt[9];
sqlite3_stmt *aStmt[10];
};
@ -43,6 +43,8 @@ struct Fts5Storage {
#define FTS5_STMT_LOOKUP_DOCSIZE 8
#define FTS5_STMT_REPLACE_CONFIG 9
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
@ -70,6 +72,8 @@ static int fts5StorageGetStmt(
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
"SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
"REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
};
Fts5Config *pConfig = p->pConfig;
char *zSql = 0;
@ -131,11 +135,13 @@ int sqlite3Fts5CreateTable(
Fts5Config *pConfig, /* FTS5 configuration */
const char *zPost, /* Shadow table to create (e.g. "content") */
const char *zDefn, /* Columns etc. for shadow table */
int bWithout, /* True for without rowid */
char **pzErr /* OUT: Error message */
){
int rc;
char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)",
pConfig->zDb, pConfig->zName, zPost, zDefn
char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s",
pConfig->zDb, pConfig->zName, zPost, zDefn,
(bWithout ? " WITHOUT ROWID" :"")
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
@ -193,12 +199,17 @@ int sqlite3Fts5StorageOpen(
for(i=0; i<pConfig->nCol; i++){
iOff += sprintf(&zDefn[iOff], ", c%d", i);
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, pzErr);
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
sqlite3_free(zDefn);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", pzErr
pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
);
}
}
@ -225,7 +236,8 @@ int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
/* If required, remove the shadow tables from the database */
if( bDestroy ){
rc = sqlite3Fts5DropTable(p->pConfig, "content");
if( rc==SQLITE_OK ) sqlite3Fts5DropTable(p->pConfig, "docsize");
if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
}
sqlite3_free(p);
@ -744,3 +756,20 @@ int sqlite3Fts5StorageRollback(Fts5Storage *p){
return sqlite3Fts5IndexRollback(p->pIndex);
}
int sqlite3Fts5StorageConfigValue(
Fts5Storage *p,
const char *z,
sqlite3_value *pVal
){
sqlite3_stmt *pReplace = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
sqlite3_bind_value(pReplace, 2, pVal);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
}
return rc;
}