mirror of
https://github.com/sqlite/sqlite.git
synced 2026-01-06 08:01:16 +03:00
Merge the fts5-btree-index branch with this one. FTS5 changes only.
FossilOrigin-Name: 9f1dcdc3e1644c21762dacf619fac70afe6318c5
This commit is contained in:
@@ -148,7 +148,7 @@ struct Fts5Config {
|
||||
};
|
||||
|
||||
/* Current expected value of %_config table 'version' field */
|
||||
#define FTS5_CURRENT_VERSION 2
|
||||
#define FTS5_CURRENT_VERSION 3
|
||||
|
||||
#define FTS5_CONTENT_NORMAL 0
|
||||
#define FTS5_CONTENT_NONE 1
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
#include "fts5Int.h"
|
||||
|
||||
#define FTS5_DEFAULT_PAGE_SIZE 1000
|
||||
#define FTS5_DEFAULT_PAGE_SIZE 4050
|
||||
#define FTS5_DEFAULT_AUTOMERGE 4
|
||||
#define FTS5_DEFAULT_CRISISMERGE 16
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ void sqlite3Fts5HashClear(Fts5Hash *pHash){
|
||||
pHash->nEntry = 0;
|
||||
}
|
||||
|
||||
static unsigned int fts5HashKey(int nSlot, const char *p, int n){
|
||||
static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
|
||||
int i;
|
||||
unsigned int h = 13;
|
||||
for(i=n-1; i>=0; i--){
|
||||
@@ -142,7 +142,7 @@ static unsigned int fts5HashKey(int nSlot, const char *p, int n){
|
||||
return (h % nSlot);
|
||||
}
|
||||
|
||||
static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){
|
||||
static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
|
||||
int i;
|
||||
unsigned int h = 13;
|
||||
for(i=n-1; i>=0; i--){
|
||||
@@ -170,7 +170,7 @@ static int fts5HashResize(Fts5Hash *pHash){
|
||||
int iHash;
|
||||
Fts5HashEntry *p = apOld[i];
|
||||
apOld[i] = p->pHashNext;
|
||||
iHash = fts5HashKey(nNew, p->zKey, strlen(p->zKey));
|
||||
iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey));
|
||||
p->pHashNext = apNew[iHash];
|
||||
apNew[iHash] = p;
|
||||
}
|
||||
@@ -210,12 +210,13 @@ int sqlite3Fts5HashWrite(
|
||||
char bByte, /* First byte of token */
|
||||
const char *pToken, int nToken /* Token to add or remove to or from index */
|
||||
){
|
||||
unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken);
|
||||
unsigned int iHash;
|
||||
Fts5HashEntry *p;
|
||||
u8 *pPtr;
|
||||
int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
|
||||
|
||||
/* Attempt to locate an existing hash entry */
|
||||
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
|
||||
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
|
||||
if( p->zKey[0]==bByte
|
||||
&& memcmp(&p->zKey[1], pToken, nToken)==0
|
||||
@@ -233,7 +234,7 @@ int sqlite3Fts5HashWrite(
|
||||
if( (pHash->nEntry*2)>=pHash->nSlot ){
|
||||
int rc = fts5HashResize(pHash);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken);
|
||||
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
|
||||
}
|
||||
|
||||
p = (Fts5HashEntry*)sqlite3_malloc(nByte);
|
||||
@@ -242,7 +243,7 @@ int sqlite3Fts5HashWrite(
|
||||
p->nAlloc = nByte;
|
||||
p->zKey[0] = bByte;
|
||||
memcpy(&p->zKey[1], pToken, nToken);
|
||||
assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) );
|
||||
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
|
||||
p->zKey[nToken+1] = '\0';
|
||||
p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
|
||||
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
|
||||
@@ -414,7 +415,7 @@ int sqlite3Fts5HashQuery(
|
||||
const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
|
||||
int *pnDoclist /* OUT: Size of doclist in bytes */
|
||||
){
|
||||
unsigned int iHash = fts5HashKey(pHash->nSlot, pTerm, nTerm);
|
||||
unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
|
||||
Fts5HashEntry *p;
|
||||
|
||||
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
|
||||
|
||||
@@ -287,9 +287,8 @@ int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
|
||||
** without overreading if the records are corrupt.
|
||||
*/
|
||||
#define FTS5_DATA_ZERO_PADDING 8
|
||||
#define FTS5_DATA_PADDING 20
|
||||
|
||||
typedef struct Fts5BtreeIter Fts5BtreeIter;
|
||||
typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
|
||||
typedef struct Fts5Data Fts5Data;
|
||||
typedef struct Fts5DlidxIter Fts5DlidxIter;
|
||||
typedef struct Fts5DlidxLvl Fts5DlidxLvl;
|
||||
@@ -333,6 +332,9 @@ struct Fts5Index {
|
||||
sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
|
||||
sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
|
||||
sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
|
||||
sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
|
||||
sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */
|
||||
sqlite3_stmt *pIdxSelect;
|
||||
int nRead; /* Total number of blocks read */
|
||||
};
|
||||
|
||||
@@ -387,8 +389,7 @@ struct Fts5DlidxWriter {
|
||||
};
|
||||
struct Fts5SegWriter {
|
||||
int iSegid; /* Segid to write to */
|
||||
int nWriter; /* Number of entries in aWriter */
|
||||
Fts5PageWriter *aWriter; /* Array of PageWriter objects */
|
||||
Fts5PageWriter writer; /* PageWriter object */
|
||||
i64 iPrevRowid; /* Previous docid written to current leaf */
|
||||
u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */
|
||||
u8 bFirstRowidInPage; /* True if next rowid is first in page */
|
||||
@@ -398,6 +399,10 @@ struct Fts5SegWriter {
|
||||
|
||||
int nDlidx; /* Allocated size of aDlidx[] array */
|
||||
Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */
|
||||
|
||||
/* Values to insert into the %_idx table */
|
||||
Fts5Buffer btterm; /* Next term to insert into %_idx table */
|
||||
int iBtPage; /* Page number corresponding to btterm */
|
||||
};
|
||||
|
||||
/*
|
||||
@@ -570,43 +575,6 @@ struct Fts5DlidxIter {
|
||||
|
||||
|
||||
|
||||
/*
|
||||
** An Fts5BtreeIter object is used to iterate through all entries in the
|
||||
** b-tree hierarchy belonging to a single fts5 segment. In this case the
|
||||
** "b-tree hierarchy" is all b-tree nodes except leaves. Each entry in the
|
||||
** b-tree hierarchy consists of the following:
|
||||
**
|
||||
** iLeaf: The page number of the leaf page the entry points to.
|
||||
**
|
||||
** term: A split-key that all terms on leaf page $iLeaf must be greater
|
||||
** than or equal to. The "term" associated with the first b-tree
|
||||
** hierarchy entry (the one that points to leaf page 1) is always
|
||||
** an empty string.
|
||||
**
|
||||
** nEmpty: The number of empty (termless) leaf pages that immediately
|
||||
** following iLeaf.
|
||||
**
|
||||
** The Fts5BtreeIter object is only used as part of the integrity-check code.
|
||||
*/
|
||||
struct Fts5BtreeIterLevel {
|
||||
Fts5NodeIter s; /* Iterator for the current node */
|
||||
Fts5Data *pData; /* Data for the current node */
|
||||
};
|
||||
struct Fts5BtreeIter {
|
||||
Fts5Index *p; /* FTS5 backend object */
|
||||
Fts5StructureSegment *pSeg; /* Iterate through this segment's b-tree */
|
||||
int nLvl; /* Size of aLvl[] array */
|
||||
Fts5BtreeIterLevel *aLvl; /* Level for each tier of b-tree */
|
||||
|
||||
/* Output variables */
|
||||
Fts5Buffer term; /* Current term */
|
||||
int iLeaf; /* Leaf containing terms >= current term */
|
||||
int nEmpty; /* Number of "empty" leaves following iLeaf */
|
||||
int bEof; /* Set to true at EOF */
|
||||
int bDlidx; /* True if there exists a dlidx */
|
||||
};
|
||||
|
||||
|
||||
/*
|
||||
** The first argument passed to this macro is a pointer to an Fts5Buffer
|
||||
** object.
|
||||
@@ -748,7 +716,7 @@ static Fts5Data *fts5DataReadOrBuffer(
|
||||
rc = SQLITE_NOMEM;
|
||||
}
|
||||
}else{
|
||||
int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
|
||||
int nSpace = nByte + FTS5_DATA_PADDING;
|
||||
pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
|
||||
if( pRet ){
|
||||
pRet->n = nByte;
|
||||
@@ -805,6 +773,23 @@ static void fts5DataRelease(Fts5Data *pData){
|
||||
sqlite3_free(pData);
|
||||
}
|
||||
|
||||
static int fts5IndexPrepareStmt(
|
||||
Fts5Index *p,
|
||||
sqlite3_stmt **ppStmt,
|
||||
char *zSql
|
||||
){
|
||||
if( p->rc==SQLITE_OK ){
|
||||
if( zSql ){
|
||||
p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
|
||||
}else{
|
||||
p->rc = SQLITE_NOMEM;
|
||||
}
|
||||
}
|
||||
sqlite3_free(zSql);
|
||||
return p->rc;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
** INSERT OR REPLACE a record into the %_data table.
|
||||
*/
|
||||
@@ -814,17 +799,11 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
|
||||
if( p->pWriter==0 ){
|
||||
int rc = SQLITE_OK;
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
char *zSql = sqlite3Fts5Mprintf(&rc,
|
||||
"REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl
|
||||
);
|
||||
if( zSql ){
|
||||
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pWriter, 0);
|
||||
sqlite3_free(zSql);
|
||||
}
|
||||
if( rc!=SQLITE_OK ){
|
||||
p->rc = rc;
|
||||
return;
|
||||
}
|
||||
fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf(
|
||||
"REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)",
|
||||
pConfig->zDb, pConfig->zName
|
||||
));
|
||||
if( p->rc ) return;
|
||||
}
|
||||
|
||||
sqlite3_bind_int64(p->pWriter, 1, iRowid);
|
||||
@@ -845,7 +824,8 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
|
||||
int rc;
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
char *zSql = sqlite3_mprintf(
|
||||
"DELETE FROM '%q'.%Q WHERE id>=? AND id<=?", pConfig->zDb, p->zDataTbl
|
||||
"DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?",
|
||||
pConfig->zDb, pConfig->zName
|
||||
);
|
||||
if( zSql==0 ){
|
||||
rc = SQLITE_NOMEM;
|
||||
@@ -872,6 +852,18 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
|
||||
i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0);
|
||||
i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1;
|
||||
fts5DataDelete(p, iFirst, iLast);
|
||||
if( p->pIdxDeleter==0 ){
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
|
||||
"DELETE FROM '%q'.'%q_idx' WHERE segid=?",
|
||||
pConfig->zDb, pConfig->zName
|
||||
));
|
||||
}
|
||||
if( p->rc==SQLITE_OK ){
|
||||
sqlite3_bind_int(p->pIdxDeleter, 1, iSegid);
|
||||
sqlite3_step(p->pIdxDeleter);
|
||||
p->rc = sqlite3_reset(p->pIdxDeleter);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -2334,12 +2326,23 @@ static void fts5SegIterSeekInit(
|
||||
|
||||
/* This block sets stack variable iPg to the leaf page number that may
|
||||
** contain term (pTerm/nTerm), if it is present in the segment. */
|
||||
for(h=pSeg->nHeight-1; h>0; h--){
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
|
||||
fts5DataBuffer(p, pBuf, iRowid);
|
||||
if( p->rc ) break;
|
||||
iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx);
|
||||
if( p->pIdxSelect==0 ){
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
|
||||
"SELECT pgno FROM '%q'.'%q_idx' WHERE "
|
||||
"segid=? AND term<=? ORDER BY term DESC LIMIT 1",
|
||||
pConfig->zDb, pConfig->zName
|
||||
));
|
||||
}
|
||||
if( p->rc ) return;
|
||||
sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
|
||||
sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
|
||||
if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
|
||||
i64 val = sqlite3_column_int(p->pIdxSelect, 0);
|
||||
iPg = (val>>1);
|
||||
bDlidx = (val & 0x0001);
|
||||
}
|
||||
p->rc = sqlite3_reset(p->pIdxSelect);
|
||||
|
||||
if( iPg<pSeg->pgnoFirst ){
|
||||
iPg = pSeg->pgnoFirst;
|
||||
@@ -3169,52 +3172,51 @@ static int fts5WriteDlidxGrow(
|
||||
}
|
||||
|
||||
/*
|
||||
** If an "nEmpty" record must be written to the b-tree before the next
|
||||
** term, write it now.
|
||||
** If the current doclist-index accumulating in pWriter->aDlidx[] is large
|
||||
** enough, flush it to disk and return 1. Otherwise discard it and return
|
||||
** zero.
|
||||
*/
|
||||
static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
if( pWriter->nEmpty ){
|
||||
int bFlag = 0;
|
||||
Fts5PageWriter *pPg;
|
||||
pPg = &pWriter->aWriter[1];
|
||||
static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
int bFlag = 0;
|
||||
|
||||
/* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
|
||||
** to the database, also write the doclist-index to disk. */
|
||||
if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
|
||||
bFlag = 1;
|
||||
}
|
||||
fts5WriteDlidxClear(p, pWriter, bFlag);
|
||||
fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag);
|
||||
fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty);
|
||||
pWriter->nEmpty = 0;
|
||||
}else{
|
||||
fts5WriteDlidxClear(p, pWriter, 0);
|
||||
/* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
|
||||
** to the database, also write the doclist-index to disk. */
|
||||
if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
|
||||
bFlag = 1;
|
||||
}
|
||||
|
||||
assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].buf.n==0 );
|
||||
assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].bPrevValid==0 );
|
||||
fts5WriteDlidxClear(p, pWriter, bFlag);
|
||||
pWriter->nEmpty = 0;
|
||||
return bFlag;
|
||||
}
|
||||
|
||||
static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
/*
|
||||
** This function is called whenever processing of the doclist for the
|
||||
** last term on leaf page (pWriter->iBtPage) is completed.
|
||||
**
|
||||
** The doclist-index for that term is currently stored in-memory within the
|
||||
** Fts5SegWriter.aDlidx[] array. If it is large enough, this function
|
||||
** writes it out to disk. Or, if it is too small to bother with, discards
|
||||
** it.
|
||||
**
|
||||
** Fts5SegWriter.btterm currently contains the first term on page iBtPage.
|
||||
*/
|
||||
static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
int bFlag;
|
||||
|
||||
assert( pWriter->iBtPage || pWriter->nEmpty==0 );
|
||||
if( pWriter->iBtPage==0 ) return;
|
||||
bFlag = fts5WriteFlushDlidx(p, pWriter);
|
||||
|
||||
if( p->rc==SQLITE_OK ){
|
||||
Fts5PageWriter *aNew;
|
||||
Fts5PageWriter *pNew;
|
||||
int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
|
||||
|
||||
aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
|
||||
if( aNew==0 ){
|
||||
p->rc = SQLITE_NOMEM;
|
||||
return;
|
||||
}
|
||||
|
||||
pNew = &aNew[pWriter->nWriter];
|
||||
memset(pNew, 0, sizeof(Fts5PageWriter));
|
||||
pNew->pgno = 1;
|
||||
fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
|
||||
|
||||
pWriter->nWriter++;
|
||||
pWriter->aWriter = aNew;
|
||||
const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
|
||||
/* The following was already done in fts5WriteInit(): */
|
||||
/* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
|
||||
sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
|
||||
sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
|
||||
sqlite3_step(p->pIdxWriter);
|
||||
p->rc = sqlite3_reset(p->pIdxWriter);
|
||||
}
|
||||
pWriter->iBtPage = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3231,36 +3233,9 @@ static void fts5WriteBtreeTerm(
|
||||
Fts5SegWriter *pWriter, /* Writer object */
|
||||
int nTerm, const u8 *pTerm /* First term on new page */
|
||||
){
|
||||
int iHeight;
|
||||
for(iHeight=1; 1; iHeight++){
|
||||
Fts5PageWriter *pPage;
|
||||
|
||||
if( iHeight>=pWriter->nWriter ){
|
||||
fts5WriteBtreeGrow(p, pWriter);
|
||||
if( p->rc ) return;
|
||||
}
|
||||
pPage = &pWriter->aWriter[iHeight];
|
||||
|
||||
fts5WriteBtreeNEmpty(p, pWriter);
|
||||
|
||||
if( pPage->buf.n>=p->pConfig->pgsz ){
|
||||
/* pPage will be written to disk. The term will be written into the
|
||||
** parent of pPage. */
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno);
|
||||
fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
|
||||
fts5BufferZero(&pPage->buf);
|
||||
fts5BufferZero(&pPage->term);
|
||||
fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno);
|
||||
pPage->pgno++;
|
||||
}else{
|
||||
int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
|
||||
fts5BufferAppendVarint(&p->rc, &pPage->buf, nPre+2);
|
||||
fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm-nPre);
|
||||
fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm-nPre, pTerm+nPre);
|
||||
fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);
|
||||
break;
|
||||
}
|
||||
}
|
||||
fts5WriteFlushBtree(p, pWriter);
|
||||
fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm);
|
||||
pWriter->iBtPage = pWriter->writer.pgno;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3344,7 +3319,7 @@ static void fts5WriteDlidxAppend(
|
||||
if( pDlidx->bPrevValid ){
|
||||
iVal = iRowid - pDlidx->iPrev;
|
||||
}else{
|
||||
i64 iPgno = (i==0 ? pWriter->aWriter[0].pgno : pDlidx[-1].pgno);
|
||||
i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
|
||||
assert( pDlidx->buf.n==0 );
|
||||
sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
|
||||
sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
|
||||
@@ -3359,7 +3334,7 @@ static void fts5WriteDlidxAppend(
|
||||
|
||||
static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
|
||||
Fts5PageWriter *pPage = &pWriter->aWriter[0];
|
||||
Fts5PageWriter *pPage = &pWriter->writer;
|
||||
i64 iRowid;
|
||||
|
||||
if( pWriter->bFirstTermInPage ){
|
||||
@@ -3398,7 +3373,7 @@ static void fts5WriteAppendTerm(
|
||||
int nTerm, const u8 *pTerm
|
||||
){
|
||||
int nPrefix; /* Bytes of prefix compression for term */
|
||||
Fts5PageWriter *pPage = &pWriter->aWriter[0];
|
||||
Fts5PageWriter *pPage = &pWriter->writer;
|
||||
|
||||
assert( pPage->buf.n==0 || pPage->buf.n>4 );
|
||||
if( pPage->buf.n==0 ){
|
||||
@@ -3433,7 +3408,7 @@ static void fts5WriteAppendTerm(
|
||||
n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
|
||||
}
|
||||
fts5WriteBtreeTerm(p, pWriter, n, pTerm);
|
||||
pPage = &pWriter->aWriter[0];
|
||||
pPage = &pWriter->writer;
|
||||
}
|
||||
}else{
|
||||
nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
|
||||
@@ -3471,7 +3446,7 @@ static void fts5WriteAppendRowid(
|
||||
int nPos
|
||||
){
|
||||
if( p->rc==SQLITE_OK ){
|
||||
Fts5PageWriter *pPage = &pWriter->aWriter[0];
|
||||
Fts5PageWriter *pPage = &pWriter->writer;
|
||||
|
||||
/* If this is to be the first docid written to the page, set the
|
||||
** docid-pointer in the page-header. Also append a value to the dlidx
|
||||
@@ -3506,7 +3481,7 @@ static void fts5WriteAppendPoslistData(
|
||||
const u8 *aData,
|
||||
int nData
|
||||
){
|
||||
Fts5PageWriter *pPage = &pWriter->aWriter[0];
|
||||
Fts5PageWriter *pPage = &pWriter->writer;
|
||||
const u8 *a = aData;
|
||||
int n = nData;
|
||||
|
||||
@@ -3529,7 +3504,7 @@ static void fts5WriteAppendPoslistData(
|
||||
}
|
||||
|
||||
static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){
|
||||
fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0);
|
||||
fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -3543,8 +3518,8 @@ static void fts5WriteFinish(
|
||||
int *pnLeaf /* OUT: Number of leaf pages in b-tree */
|
||||
){
|
||||
int i;
|
||||
Fts5PageWriter *pLeaf = &pWriter->writer;
|
||||
if( p->rc==SQLITE_OK ){
|
||||
Fts5PageWriter *pLeaf = &pWriter->aWriter[0];
|
||||
if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){
|
||||
*pnLeaf = 0;
|
||||
*pnHeight = 0;
|
||||
@@ -3553,29 +3528,14 @@ static void fts5WriteFinish(
|
||||
fts5WriteFlushLeaf(p, pWriter);
|
||||
}
|
||||
*pnLeaf = pLeaf->pgno-1;
|
||||
if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
|
||||
fts5WriteBtreeGrow(p, pWriter);
|
||||
}
|
||||
if( pWriter->nWriter>1 ){
|
||||
fts5WriteBtreeNEmpty(p, pWriter);
|
||||
}
|
||||
*pnHeight = pWriter->nWriter;
|
||||
|
||||
for(i=1; i<pWriter->nWriter; i++){
|
||||
Fts5PageWriter *pPg = &pWriter->aWriter[i];
|
||||
fts5DataWrite(p,
|
||||
FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno),
|
||||
pPg->buf.p, pPg->buf.n
|
||||
);
|
||||
}
|
||||
fts5WriteFlushBtree(p, pWriter);
|
||||
*pnHeight = 0;
|
||||
}
|
||||
}
|
||||
for(i=0; i<pWriter->nWriter; i++){
|
||||
Fts5PageWriter *pPg = &pWriter->aWriter[i];
|
||||
fts5BufferFree(&pPg->term);
|
||||
fts5BufferFree(&pPg->buf);
|
||||
}
|
||||
sqlite3_free(pWriter->aWriter);
|
||||
fts5BufferFree(&pLeaf->term);
|
||||
fts5BufferFree(&pLeaf->buf);
|
||||
fts5BufferFree(&pWriter->btterm);
|
||||
|
||||
for(i=0; i<pWriter->nDlidx; i++){
|
||||
sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf);
|
||||
@@ -3591,48 +3551,21 @@ static void fts5WriteInit(
|
||||
memset(pWriter, 0, sizeof(Fts5SegWriter));
|
||||
pWriter->iSegid = iSegid;
|
||||
|
||||
pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter));
|
||||
if( fts5WriteDlidxGrow(p, pWriter, 1) ) return;
|
||||
pWriter->nWriter = 1;
|
||||
pWriter->nDlidx = 1;
|
||||
pWriter->aWriter[0].pgno = 1;
|
||||
fts5WriteDlidxGrow(p, pWriter, 1);
|
||||
pWriter->writer.pgno = 1;
|
||||
pWriter->bFirstTermInPage = 1;
|
||||
}
|
||||
pWriter->iBtPage = 1;
|
||||
|
||||
static void fts5WriteInitForAppend(
|
||||
Fts5Index *p, /* FTS5 backend object */
|
||||
Fts5SegWriter *pWriter, /* Writer to initialize */
|
||||
Fts5StructureSegment *pSeg /* Segment object to append to */
|
||||
){
|
||||
int nByte = pSeg->nHeight * sizeof(Fts5PageWriter);
|
||||
memset(pWriter, 0, sizeof(Fts5SegWriter));
|
||||
pWriter->iSegid = pSeg->iSegid;
|
||||
pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte);
|
||||
pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter));
|
||||
if( p->pIdxWriter==0 ){
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf(
|
||||
"INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)",
|
||||
pConfig->zDb, pConfig->zName
|
||||
));
|
||||
}
|
||||
|
||||
if( p->rc==SQLITE_OK ){
|
||||
int pgno = 1;
|
||||
int i;
|
||||
pWriter->nDlidx = 1;
|
||||
pWriter->nWriter = pSeg->nHeight;
|
||||
pWriter->aWriter[0].pgno = pSeg->pgnoLast+1;
|
||||
for(i=pSeg->nHeight-1; i>0; i--){
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno);
|
||||
Fts5PageWriter *pPg = &pWriter->aWriter[i];
|
||||
pPg->pgno = pgno;
|
||||
fts5DataBuffer(p, &pPg->buf, iRowid);
|
||||
if( p->rc==SQLITE_OK ){
|
||||
Fts5NodeIter ss;
|
||||
fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss);
|
||||
while( ss.aData ) fts5NodeIterNext(&p->rc, &ss);
|
||||
fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p);
|
||||
pgno = ss.iChild;
|
||||
fts5NodeIterFree(&ss);
|
||||
}
|
||||
}
|
||||
assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast );
|
||||
pWriter->bFirstTermInPage = 1;
|
||||
assert( pWriter->aWriter[0].term.n==0 );
|
||||
sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3672,7 +3605,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
|
||||
fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]);
|
||||
fts5DataRelease(pData);
|
||||
pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
|
||||
fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid);
|
||||
fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid);
|
||||
fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
|
||||
}
|
||||
}
|
||||
@@ -3719,8 +3652,11 @@ static void fts5IndexMergeLevel(
|
||||
pLvlOut = &pStruct->aLevel[iLvl+1];
|
||||
assert( pLvlOut->nSeg>0 );
|
||||
nInput = pLvl->nMerge;
|
||||
fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]);
|
||||
pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
|
||||
|
||||
fts5WriteInit(p, &writer, pSeg->iSegid);
|
||||
writer.writer.pgno = pSeg->pgnoLast+1;
|
||||
writer.iBtPage = 0;
|
||||
}else{
|
||||
int iSegid = fts5AllocateSegid(p, pStruct);
|
||||
|
||||
@@ -3811,7 +3747,7 @@ static void fts5IndexMergeLevel(
|
||||
pStruct->nSegment--;
|
||||
}
|
||||
}else{
|
||||
assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 );
|
||||
assert( pSeg->pgnoLast>0 );
|
||||
fts5TrimSegments(p, pIter);
|
||||
pLvl->nMerge = nInput;
|
||||
}
|
||||
@@ -3986,7 +3922,7 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
/* Pre-allocate the buffer used to assemble leaf pages to the target
|
||||
** page size. */
|
||||
assert( pgsz>0 );
|
||||
pBuf = &writer.aWriter[0].buf;
|
||||
pBuf = &writer.writer.buf;
|
||||
fts5BufferGrow(&p->rc, pBuf, pgsz + 20);
|
||||
|
||||
/* Begin scanning through hash table entries. This loop runs once for each
|
||||
@@ -4010,7 +3946,7 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
** flush the leaf to disk here. */
|
||||
if( (pBuf->n + nTerm + 2) > pgsz ){
|
||||
fts5WriteFlushLeaf(p, &writer);
|
||||
pBuf = &writer.aWriter[0].buf;
|
||||
pBuf = &writer.writer.buf;
|
||||
if( (nTerm + 32) > pBuf->nSpace ){
|
||||
fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
|
||||
if( p->rc ) break;
|
||||
@@ -4027,10 +3963,10 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
}else{
|
||||
fts5PutU16(&pBuf->p[2], pBuf->n);
|
||||
writer.bFirstTermInPage = 0;
|
||||
if( writer.aWriter[0].pgno!=1 ){
|
||||
if( writer.writer.pgno!=1 ){
|
||||
int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm);
|
||||
fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
|
||||
pBuf = &writer.aWriter[0].buf;
|
||||
pBuf = &writer.writer.buf;
|
||||
assert( nPre<nTerm );
|
||||
}
|
||||
nSuffix = nTerm;
|
||||
@@ -4042,7 +3978,7 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
** doclist-index is to be generated for this doclist, it will be
|
||||
** associated with this page. */
|
||||
assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 );
|
||||
writer.aDlidx[0].pgno = writer.aWriter[0].pgno;
|
||||
writer.aDlidx[0].pgno = writer.writer.pgno;
|
||||
|
||||
if( pgsz>=(pBuf->n + nDoclist + 1) ){
|
||||
/* The entire doclist will fit on the current leaf. */
|
||||
@@ -4099,7 +4035,7 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
iPos += n;
|
||||
if( pBuf->n>=pgsz ){
|
||||
fts5WriteFlushLeaf(p, &writer);
|
||||
pBuf = &writer.aWriter[0].buf;
|
||||
pBuf = &writer.writer.buf;
|
||||
}
|
||||
if( iPos>=nCopy ) break;
|
||||
}
|
||||
@@ -4133,7 +4069,6 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
fts5StructurePromote(p, 0, pStruct);
|
||||
}
|
||||
|
||||
|
||||
fts5IndexAutomerge(p, &pStruct, pgnoLast);
|
||||
fts5IndexCrisismerge(p, &pStruct);
|
||||
fts5StructureWrite(p, pStruct);
|
||||
@@ -4561,6 +4496,12 @@ int sqlite3Fts5IndexOpen(
|
||||
rc = sqlite3Fts5CreateTable(
|
||||
pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
|
||||
);
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts5CreateTable(pConfig, "idx",
|
||||
"segid, term, pgno, PRIMARY KEY(segid, term)",
|
||||
1, pzErr
|
||||
);
|
||||
}
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts5IndexReinit(p);
|
||||
}
|
||||
@@ -4584,6 +4525,9 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
|
||||
assert( p->pReader==0 );
|
||||
sqlite3_finalize(p->pWriter);
|
||||
sqlite3_finalize(p->pDeleter);
|
||||
sqlite3_finalize(p->pIdxWriter);
|
||||
sqlite3_finalize(p->pIdxDeleter);
|
||||
sqlite3_finalize(p->pIdxSelect);
|
||||
sqlite3Fts5HashFree(p->pHash);
|
||||
sqlite3Fts5BufferFree(&p->scratch);
|
||||
sqlite3_free(p->zDataTbl);
|
||||
@@ -4933,92 +4877,6 @@ static u64 fts5IndexEntryCksum(
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void fts5BtreeIterInit(
|
||||
Fts5Index *p,
|
||||
Fts5StructureSegment *pSeg,
|
||||
Fts5BtreeIter *pIter
|
||||
){
|
||||
int nByte;
|
||||
int i;
|
||||
nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
|
||||
memset(pIter, 0, sizeof(*pIter));
|
||||
if( nByte ){
|
||||
pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
|
||||
}
|
||||
if( p->rc==SQLITE_OK ){
|
||||
pIter->nLvl = pSeg->nHeight-1;
|
||||
pIter->p = p;
|
||||
pIter->pSeg = pSeg;
|
||||
}
|
||||
for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1);
|
||||
Fts5Data *pData;
|
||||
pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid);
|
||||
if( pData ){
|
||||
fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s);
|
||||
}
|
||||
}
|
||||
|
||||
if( pIter->nLvl==0 || p->rc ){
|
||||
pIter->bEof = 1;
|
||||
pIter->iLeaf = pSeg->pgnoLast;
|
||||
}else{
|
||||
pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
|
||||
pIter->iLeaf = pIter->aLvl[0].s.iChild;
|
||||
pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
|
||||
}
|
||||
}
|
||||
|
||||
static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
|
||||
Fts5Index *p = pIter->p;
|
||||
int i;
|
||||
|
||||
assert( pIter->bEof==0 && pIter->aLvl[0].s.aData );
|
||||
for(i=0; i<pIter->nLvl && p->rc==SQLITE_OK; i++){
|
||||
Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
|
||||
fts5NodeIterNext(&p->rc, &pLvl->s);
|
||||
if( pLvl->s.aData ){
|
||||
fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p);
|
||||
break;
|
||||
}else{
|
||||
fts5NodeIterFree(&pLvl->s);
|
||||
fts5DataRelease(pLvl->pData);
|
||||
pLvl->pData = 0;
|
||||
}
|
||||
}
|
||||
if( i==pIter->nLvl || p->rc ){
|
||||
pIter->bEof = 1;
|
||||
}else{
|
||||
int iSegid = pIter->pSeg->iSegid;
|
||||
for(i--; i>=0; i--){
|
||||
Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild);
|
||||
pLvl->pData = fts5DataRead(p, iRowid);
|
||||
if( pLvl->pData ){
|
||||
fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
|
||||
pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
|
||||
pIter->iLeaf = pIter->aLvl[0].s.iChild;
|
||||
}
|
||||
|
||||
static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
|
||||
int i;
|
||||
for(i=0; i<pIter->nLvl; i++){
|
||||
Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
|
||||
fts5NodeIterFree(&pLvl->s);
|
||||
if( pLvl->pData ){
|
||||
fts5DataRelease(pLvl->pData);
|
||||
pLvl->pData = 0;
|
||||
}
|
||||
}
|
||||
sqlite3_free(pIter->aLvl);
|
||||
fts5BufferFree(&pIter->term);
|
||||
}
|
||||
|
||||
#ifdef SQLITE_DEBUG
|
||||
/*
|
||||
** This function is purely an internal test. It does not contribute to
|
||||
@@ -5166,33 +5024,74 @@ static void fts5TestTerm(
|
||||
# define fts5TestTerm(u,v,w,x,y,z)
|
||||
#endif
|
||||
|
||||
/*
|
||||
** Check that:
|
||||
**
|
||||
** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and
|
||||
** contain zero terms.
|
||||
** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and
|
||||
** contain zero rowids.
|
||||
*/
|
||||
static void fts5IndexIntegrityCheckEmpty(
|
||||
Fts5Index *p,
|
||||
Fts5StructureSegment *pSeg, /* Segment to check internal consistency */
|
||||
int iFirst,
|
||||
int iNoRowid,
|
||||
int iLast
|
||||
){
|
||||
int i;
|
||||
|
||||
/* Now check that the iter.nEmpty leaves following the current leaf
|
||||
** (a) exist and (b) contain no terms. */
|
||||
for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){
|
||||
Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i));
|
||||
if( pLeaf ){
|
||||
if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT;
|
||||
if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
fts5DataRelease(pLeaf);
|
||||
if( p->rc ) break;
|
||||
}
|
||||
}
|
||||
|
||||
static void fts5IndexIntegrityCheckSegment(
|
||||
Fts5Index *p, /* FTS5 backend object */
|
||||
Fts5StructureSegment *pSeg /* Segment to check internal consistency */
|
||||
){
|
||||
Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
sqlite3_stmt *pStmt = 0;
|
||||
int rc2;
|
||||
int iIdxPrevLeaf = pSeg->pgnoFirst-1;
|
||||
int iDlidxPrevLeaf = pSeg->pgnoLast;
|
||||
|
||||
if( pSeg->pgnoFirst==0 ) return;
|
||||
|
||||
fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf(
|
||||
"SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d",
|
||||
pConfig->zDb, pConfig->zName, pSeg->iSegid
|
||||
));
|
||||
|
||||
/* Iterate through the b-tree hierarchy. */
|
||||
for(fts5BtreeIterInit(p, pSeg, &iter);
|
||||
p->rc==SQLITE_OK && iter.bEof==0;
|
||||
fts5BtreeIterNext(&iter)
|
||||
){
|
||||
while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
i64 iRow; /* Rowid for this leaf */
|
||||
Fts5Data *pLeaf; /* Data for this leaf */
|
||||
int iOff; /* Offset of first term on leaf */
|
||||
int i; /* Used to iterate through empty leaves */
|
||||
|
||||
int nIdxTerm = sqlite3_column_bytes(pStmt, 1);
|
||||
const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1);
|
||||
int iIdxLeaf = sqlite3_column_int(pStmt, 2);
|
||||
int bIdxDlidx = sqlite3_column_int(pStmt, 3);
|
||||
|
||||
/* If the leaf in question has already been trimmed from the segment,
|
||||
** ignore this b-tree entry. Otherwise, load it into memory. */
|
||||
if( iter.iLeaf<pSeg->pgnoFirst ) continue;
|
||||
iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf);
|
||||
if( iIdxLeaf<pSeg->pgnoFirst ) continue;
|
||||
iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf);
|
||||
pLeaf = fts5DataRead(p, iRow);
|
||||
if( pLeaf==0 ) break;
|
||||
|
||||
/* Check that the leaf contains at least one term, and that it is equal
|
||||
** to or larger than the split-key in iter.term. Also check that if there
|
||||
** to or larger than the split-key in zIdxTerm. Also check that if there
|
||||
** is also a rowid pointer within the leaf page header, it points to a
|
||||
** location before the term. */
|
||||
iOff = fts5GetU16(&pLeaf->p[2]);
|
||||
@@ -5208,8 +5107,8 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}else{
|
||||
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
|
||||
res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
|
||||
if( res==0 ) res = nTerm - iter.term.n;
|
||||
res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm));
|
||||
if( res==0 ) res = nTerm - nIdxTerm;
|
||||
if( res<0 ) p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
}
|
||||
@@ -5219,23 +5118,20 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
|
||||
/* Now check that the iter.nEmpty leaves following the current leaf
|
||||
** (a) exist and (b) contain no terms. */
|
||||
for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
|
||||
pLeaf = fts5DataRead(p, iRow+i);
|
||||
if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
fts5DataRelease(pLeaf);
|
||||
}
|
||||
fts5IndexIntegrityCheckEmpty(
|
||||
p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1
|
||||
);
|
||||
if( p->rc ) break;
|
||||
|
||||
/* If there is a doclist-index, check that it looks right. */
|
||||
if( iter.bDlidx ){
|
||||
if( bIdxDlidx ){
|
||||
Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
|
||||
int iPrevLeaf = iter.iLeaf;
|
||||
int iPrevLeaf = iIdxLeaf;
|
||||
int iSegid = pSeg->iSegid;
|
||||
int iPg;
|
||||
i64 iKey;
|
||||
|
||||
for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf);
|
||||
for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf);
|
||||
fts5DlidxIterEof(p, pDlidx)==0;
|
||||
fts5DlidxIterNext(p, pDlidx)
|
||||
){
|
||||
@@ -5268,26 +5164,26 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
}
|
||||
}
|
||||
|
||||
for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){
|
||||
iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg);
|
||||
pLeaf = fts5DataRead(p, iKey);
|
||||
if( pLeaf ){
|
||||
if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
|
||||
fts5DataRelease(pLeaf);
|
||||
}
|
||||
}
|
||||
|
||||
iDlidxPrevLeaf = iPg;
|
||||
fts5DlidxIterFree(pDlidx);
|
||||
fts5TestDlidxReverse(p, iSegid, iter.iLeaf);
|
||||
fts5TestDlidxReverse(p, iSegid, iIdxLeaf);
|
||||
}else{
|
||||
iDlidxPrevLeaf = pSeg->pgnoLast;
|
||||
/* TODO: Check there is no doclist index */
|
||||
}
|
||||
|
||||
iIdxPrevLeaf = iIdxLeaf;
|
||||
}
|
||||
|
||||
rc2 = sqlite3_finalize(pStmt);
|
||||
if( p->rc==SQLITE_OK ) p->rc = rc2;
|
||||
|
||||
/* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
|
||||
#if 0
|
||||
if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
|
||||
fts5BtreeIterFree(&iter);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -180,8 +180,10 @@ static int fts5ExecPrintf(
|
||||
int sqlite3Fts5DropAll(Fts5Config *pConfig){
|
||||
int rc = fts5ExecPrintf(pConfig->db, 0,
|
||||
"DROP TABLE IF EXISTS %Q.'%q_data';"
|
||||
"DROP TABLE IF EXISTS %Q.'%q_idx';"
|
||||
"DROP TABLE IF EXISTS %Q.'%q_config';",
|
||||
pConfig->zDb, pConfig->zName,
|
||||
pConfig->zDb, pConfig->zName,
|
||||
pConfig->zDb, pConfig->zName
|
||||
);
|
||||
if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
||||
@@ -218,6 +220,7 @@ int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
|
||||
int rc = sqlite3Fts5StorageSync(pStorage, 1);
|
||||
|
||||
fts5StorageRenameOne(pConfig, &rc, "data", zName);
|
||||
fts5StorageRenameOne(pConfig, &rc, "idx", zName);
|
||||
fts5StorageRenameOne(pConfig, &rc, "config", zName);
|
||||
if( pConfig->bColumnsize ){
|
||||
fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
|
||||
@@ -601,7 +604,9 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
|
||||
|
||||
/* Delete the contents of the %_data and %_docsize tables. */
|
||||
rc = fts5ExecPrintf(pConfig->db, 0,
|
||||
"DELETE FROM %Q.'%q_data';",
|
||||
"DELETE FROM %Q.'%q_data';"
|
||||
"DELETE FROM %Q.'%q_idx';",
|
||||
pConfig->zDb, pConfig->zName,
|
||||
pConfig->zDb, pConfig->zName
|
||||
);
|
||||
if( rc==SQLITE_OK && pConfig->bColumnsize ){
|
||||
|
||||
@@ -27,6 +27,7 @@ do_execsql_test 1.0 {
|
||||
} {
|
||||
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
|
||||
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
|
||||
t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID}
|
||||
t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
|
||||
t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
|
||||
t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
|
||||
@@ -47,9 +48,10 @@ do_execsql_test 2.0 {
|
||||
do_execsql_test 2.1 {
|
||||
INSERT INTO t1 VALUES('a b c', 'd e f');
|
||||
}
|
||||
|
||||
do_test 2.2 {
|
||||
execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
|
||||
} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/}
|
||||
} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/}
|
||||
|
||||
foreach w {a b c d e f} {
|
||||
do_execsql_test 2.3.$w.asc {
|
||||
|
||||
@@ -26,17 +26,17 @@ ifcapable !fts5 {
|
||||
do_execsql_test 1.1 {
|
||||
CREATE VIRTUAL TABLE ft1 USING fts5(x);
|
||||
SELECT * FROM ft1_config;
|
||||
} {version 2}
|
||||
} {version 3}
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
|
||||
SELECT * FROM ft1_config;
|
||||
} {pgsz 32 version 2}
|
||||
} {pgsz 32 version 3}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
|
||||
SELECT * FROM ft1_config;
|
||||
} {pgsz 64 version 2}
|
||||
} {pgsz 64 version 3}
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
# Test the logic for parsing the rank() function definition.
|
||||
|
||||
@@ -247,7 +247,7 @@ reset_db
|
||||
do_execsql_test 6.1 {
|
||||
CREATE VIRTUAL TABLE xx USING fts5(x, y, content="");
|
||||
SELECT name FROM sqlite_master;
|
||||
} {xx xx_data xx_docsize xx_config}
|
||||
} {xx xx_data xx_idx xx_docsize xx_config}
|
||||
do_execsql_test 6.2 {
|
||||
DROP TABLE xx;
|
||||
SELECT name FROM sqlite_master;
|
||||
|
||||
@@ -30,34 +30,34 @@ do_execsql_test 1.1 {
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
SELECT * FROM t1_config WHERE k='version'
|
||||
} {version 2}
|
||||
} {version 3}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
SELECT rowid FROM t1 WHERE t1 MATCH 'a';
|
||||
} {1}
|
||||
|
||||
do_execsql_test 1.4 {
|
||||
UPDATE t1_config set v=3 WHERE k='version';
|
||||
UPDATE t1_config set v=4 WHERE k='version';
|
||||
}
|
||||
|
||||
do_test 1.5 {
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
|
||||
} {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}}
|
||||
|
||||
do_test 1.6 {
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { INSERT INTO t1 VALUES('x y z') }
|
||||
} {1 {invalid fts5 file format (found 3, expected 2) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}}
|
||||
|
||||
do_test 1.7 {
|
||||
execsql { DELETE FROM t1_config WHERE k='version' }
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
|
||||
} {1 {invalid fts5 file format (found 0, expected 2) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 0, expected 3) - run 'rebuild'}}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
28
manifest
28
manifest
@@ -1,5 +1,5 @@
|
||||
C Remove\sa\sduplicated\s"if"\scondition\sfrom\sfts3.c.
|
||||
D 2015-07-25T19:26:52.805
|
||||
C Merge\sthe\sfts5-btree-index\sbranch\swith\sthis\sone.\sFTS5\schanges\sonly.
|
||||
D 2015-07-27T12:19:11.594
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 4de3ef40c8b3b75c0c55ff4242a43c8ce1ad90ee
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@@ -106,15 +106,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
|
||||
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
|
||||
F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
|
||||
F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
|
||||
F ext/fts5/fts5Int.h 8d9bce1847a10df2e4ed9492ea4f3868276748fb
|
||||
F ext/fts5/fts5Int.h 4d669e2ef0f8d51380c78403fd310ee69ce0f70e
|
||||
F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0
|
||||
F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
|
||||
F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
|
||||
F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf
|
||||
F ext/fts5/fts5_expr.c 56dcbcbdc9029dd76a31360de664559839f4be41
|
||||
F ext/fts5/fts5_hash.c ff07722c73587c12781213133edbdb22cd156378
|
||||
F ext/fts5/fts5_index.c 892c13a7f44b68e962a91af62f2078f23fabb241
|
||||
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
|
||||
F ext/fts5/fts5_index.c f5b25da3a2eef71f2024a08323a1575eb55f7aad
|
||||
F ext/fts5/fts5_main.c 0de7ba81488d2c502c8e794eaf7983d468e4c6e9
|
||||
F ext/fts5/fts5_storage.c 1c35a38a564ee9cadcbd7ae0b13a806bdda722bd
|
||||
F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5
|
||||
F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
|
||||
F ext/fts5/fts5_tokenize.c 30f97a8c74683797b4cd233790444fbefb3b0708
|
||||
F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
|
||||
@@ -123,7 +123,7 @@ F ext/fts5/fts5_vocab.c 4e268a3fcbc099e50e335a1135be985a41ff6f7f
|
||||
F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
|
||||
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
|
||||
F ext/fts5/test/fts5_common.tcl e0b4a846a7670f6232a644ece69ef25a5c19c0e8
|
||||
F ext/fts5/test/fts5aa.test 4e896b9154764fed48179a87ba0bdf3650d7f49d
|
||||
F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0
|
||||
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
|
||||
F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
|
||||
F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
|
||||
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae
|
||||
F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
|
||||
F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
|
||||
F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
|
||||
F ext/fts5/test/fts5al.test fc60ebeac9d8e366e71309d4c31fa72199d711d7
|
||||
F ext/fts5/test/fts5al.test 440d77c0b39ba73bad2ceb8986c2bb1093570735
|
||||
F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04
|
||||
F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
|
||||
F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
|
||||
@@ -142,7 +142,7 @@ F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
|
||||
F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
|
||||
F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99
|
||||
F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b
|
||||
F ext/fts5/test/fts5content.test d0d90a45f0bcf07d75d474500d81f941b45e2021
|
||||
F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
|
||||
F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6
|
||||
F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b
|
||||
F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3
|
||||
@@ -175,7 +175,7 @@ F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
|
||||
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
|
||||
F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
|
||||
F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
|
||||
F ext/fts5/test/fts5version.test c54a708236642bcc850d2aedc6f505fef1d9f9f1
|
||||
F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8
|
||||
F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
|
||||
F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556
|
||||
F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
|
||||
@@ -1366,7 +1366,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
|
||||
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
|
||||
P 7efed5786bb6a7a452f5aece9d6c6a09e6930939
|
||||
R 02cd6ad3aa414fdb248eb64044a0f9b8
|
||||
P ee348b12754abda1fe13231c1868faca9d78481c 92c9178933ef95e92c343c0a741a76bbdf071966
|
||||
R 8f58f88db8a7564cc507b9707cc8c853
|
||||
U dan
|
||||
Z 8274b8233f11154aae74796194660376
|
||||
Z 9ebcb4f7d5c62fb47a944f9a469a2828
|
||||
|
||||
@@ -1 +1 @@
|
||||
ee348b12754abda1fe13231c1868faca9d78481c
|
||||
9f1dcdc3e1644c21762dacf619fac70afe6318c5
|
||||
Reference in New Issue
Block a user