mirror of
https://github.com/sqlite/sqlite.git
synced 2025-07-30 19:03:16 +03:00
Add the 'secure-delete' option to the fts5 extension. For configuring fts5 to delete old entries directly from the full-text index instead of using delete keys.
FossilOrigin-Name: 394980e4fe12125460ab14da41edae9089a4da332a46f3124bf0b9778793791f
This commit is contained in:
dan
@ -199,6 +199,7 @@ struct Fts5Config {
|
||||
int ePattern; /* FTS_PATTERN_XXX constant */
|
||||
|
||||
/* Values loaded from the %_config table */
|
||||
int iVersion; /* fts5 file format 'version' */
|
||||
int iCookie; /* Incremented when %_config is modified */
|
||||
int pgsz; /* Approximate page size used in %_data */
|
||||
int nAutomerge; /* 'automerge' setting */
|
||||
@ -207,6 +208,7 @@ struct Fts5Config {
|
||||
int nHashSize; /* Bytes of memory for in-memory hash */
|
||||
char *zRank; /* Name of rank function */
|
||||
char *zRankArgs; /* Arguments to rank function */
|
||||
int bSecureDelete; /* 'secure-delete' */
|
||||
|
||||
/* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
|
||||
char **pzErrmsg;
|
||||
@ -216,8 +218,11 @@ struct Fts5Config {
|
||||
#endif
|
||||
};
|
||||
|
||||
/* Current expected value of %_config table 'version' field */
|
||||
#define FTS5_CURRENT_VERSION 4
|
||||
/* Current expected value of %_config table 'version' field. And
|
||||
** the expected version if the 'secure-delete' option has ever been
|
||||
** set on the table. */
|
||||
#define FTS5_CURRENT_VERSION 4
|
||||
#define FTS5_CURRENT_VERSION_SECUREDELETE 5
|
||||
|
||||
#define FTS5_CONTENT_NORMAL 0
|
||||
#define FTS5_CONTENT_NONE 1
|
||||
@ -383,6 +388,7 @@ struct Fts5IndexIter {
|
||||
** above. */
|
||||
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
|
||||
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
|
||||
#define FTS5INDEX_QUERY_SKIPHASH 0x0040
|
||||
|
||||
/*
|
||||
** Create/destroy an Fts5Index object.
|
||||
|
||||
@ -903,6 +903,18 @@ int sqlite3Fts5ConfigSetValue(
|
||||
rc = SQLITE_OK;
|
||||
*pbBadkey = 1;
|
||||
}
|
||||
}
|
||||
|
||||
else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
|
||||
int bVal = -1;
|
||||
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
|
||||
bVal = sqlite3_value_int(pVal);
|
||||
}
|
||||
if( bVal<0 ){
|
||||
*pbBadkey = 1;
|
||||
}else{
|
||||
pConfig->bSecureDelete = (bVal ? 1 : 0);
|
||||
}
|
||||
}else{
|
||||
*pbBadkey = 1;
|
||||
}
|
||||
@ -947,15 +959,20 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
|
||||
rc = sqlite3_finalize(p);
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
|
||||
if( rc==SQLITE_OK
|
||||
&& iVersion!=FTS5_CURRENT_VERSION
|
||||
&& iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
|
||||
){
|
||||
rc = SQLITE_ERROR;
|
||||
if( pConfig->pzErrmsg ){
|
||||
assert( 0==*pConfig->pzErrmsg );
|
||||
*pConfig->pzErrmsg = sqlite3_mprintf(
|
||||
"invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
|
||||
iVersion, FTS5_CURRENT_VERSION
|
||||
*pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
|
||||
"(found %d, expected %d or %d) - run 'rebuild'",
|
||||
iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
|
||||
);
|
||||
}
|
||||
}else{
|
||||
pConfig->iVersion = iVersion;
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
|
||||
@ -302,6 +302,8 @@ struct Fts5Index {
|
||||
sqlite3_stmt *pIdxSelect;
|
||||
int nRead; /* Total number of blocks read */
|
||||
|
||||
sqlite3_stmt *pDeleteFromIdx;
|
||||
|
||||
sqlite3_stmt *pDataVersion;
|
||||
i64 iStructVersion; /* data_version when pStruct read */
|
||||
Fts5Structure *pStruct; /* Current db structure (or NULL) */
|
||||
@ -394,9 +396,6 @@ struct Fts5CResult {
|
||||
** iLeafOffset:
|
||||
** Byte offset within the current leaf that is the first byte of the
|
||||
** position list data (one byte passed the position-list size field).
|
||||
** rowid field of the current entry. Usually this is the size field of the
|
||||
** position list data. The exception is if the rowid for the current entry
|
||||
** is the last thing on the leaf page.
|
||||
**
|
||||
** pLeaf:
|
||||
** Buffer containing current leaf page data. Set to NULL at EOF.
|
||||
@ -1443,42 +1442,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
|
||||
pLvl->bEof = 1;
|
||||
}else{
|
||||
u8 *a = pLvl->pData->p;
|
||||
i64 iVal;
|
||||
int iLimit;
|
||||
int ii;
|
||||
int nZero = 0;
|
||||
|
||||
/* Currently iOff points to the first byte of a varint. This block
|
||||
** decrements iOff until it points to the first byte of the previous
|
||||
** varint. Taking care not to read any memory locations that occur
|
||||
** before the buffer in memory. */
|
||||
iLimit = (iOff>9 ? iOff-9 : 0);
|
||||
for(iOff--; iOff>iLimit; iOff--){
|
||||
if( (a[iOff-1] & 0x80)==0 ) break;
|
||||
}
|
||||
pLvl->iOff = 0;
|
||||
fts5DlidxLvlNext(pLvl);
|
||||
while( 1 ){
|
||||
int nZero = 0;
|
||||
int ii = pLvl->iOff;
|
||||
u64 delta = 0;
|
||||
|
||||
fts5GetVarint(&a[iOff], (u64*)&iVal);
|
||||
pLvl->iRowid -= iVal;
|
||||
pLvl->iLeafPgno--;
|
||||
|
||||
/* Skip backwards past any 0x00 varints. */
|
||||
for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){
|
||||
nZero++;
|
||||
}
|
||||
if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){
|
||||
/* The byte immediately before the last 0x00 byte has the 0x80 bit
|
||||
** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80
|
||||
** bytes before a[ii]. */
|
||||
int bZero = 0; /* True if last 0x00 counts */
|
||||
if( (ii-8)>=pLvl->iFirstOff ){
|
||||
int j;
|
||||
for(j=1; j<=8 && (a[ii-j] & 0x80); j++);
|
||||
bZero = (j>8);
|
||||
while( a[ii]==0 ){
|
||||
nZero++;
|
||||
ii++;
|
||||
}
|
||||
if( bZero==0 ) nZero--;
|
||||
ii += sqlite3Fts5GetVarint(&a[ii], &delta);
|
||||
|
||||
if( ii>=iOff ) break;
|
||||
pLvl->iLeafPgno += nZero+1;
|
||||
pLvl->iRowid += delta;
|
||||
pLvl->iOff = ii;
|
||||
}
|
||||
pLvl->iLeafPgno -= nZero;
|
||||
pLvl->iOff = iOff - nZero;
|
||||
}
|
||||
|
||||
return pLvl->bEof;
|
||||
@ -1674,7 +1656,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
|
||||
i64 iOff = pIter->iLeafOffset;
|
||||
|
||||
ASSERT_SZLEAF_OK(pIter->pLeaf);
|
||||
if( iOff>=pIter->pLeaf->szLeaf ){
|
||||
while( iOff>=pIter->pLeaf->szLeaf ){
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
if( pIter->pLeaf==0 ){
|
||||
if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
|
||||
@ -1773,10 +1755,12 @@ static void fts5SegIterInit(
|
||||
fts5SegIterSetNext(p, pIter);
|
||||
pIter->pSeg = pSeg;
|
||||
pIter->iLeafPgno = pSeg->pgnoFirst-1;
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
do {
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
}while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 );
|
||||
}
|
||||
|
||||
if( p->rc==SQLITE_OK ){
|
||||
if( p->rc==SQLITE_OK && pIter->pLeaf ){
|
||||
pIter->iLeafOffset = 4;
|
||||
assert( pIter->pLeaf!=0 );
|
||||
assert_nc( pIter->pLeaf->nn>4 );
|
||||
@ -2163,7 +2147,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
|
||||
Fts5Data *pLast = 0;
|
||||
int pgnoLast = 0;
|
||||
|
||||
if( pDlidx ){
|
||||
if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){
|
||||
int iSegid = pIter->pSeg->iSegid;
|
||||
pgnoLast = fts5DlidxIterPgno(pDlidx);
|
||||
pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
|
||||
@ -2724,7 +2708,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
|
||||
|
||||
/*
|
||||
** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
|
||||
** It is an error if leaf iLeafPgno does not exist or contains no rowids.
|
||||
** It is an error if leaf iLeafPgno does not exist. Unless the db is
|
||||
** a 'secure-delete' db, if it contains no rowids then this is also an error.
|
||||
*/
|
||||
static void fts5SegIterGotoPage(
|
||||
Fts5Index *p, /* FTS5 backend object */
|
||||
@ -2739,21 +2724,23 @@ static void fts5SegIterGotoPage(
|
||||
fts5DataRelease(pIter->pNextLeaf);
|
||||
pIter->pNextLeaf = 0;
|
||||
pIter->iLeafPgno = iLeafPgno-1;
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
|
||||
|
||||
if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){
|
||||
while( p->rc==SQLITE_OK ){
|
||||
int iOff;
|
||||
u8 *a = pIter->pLeaf->p;
|
||||
int n = pIter->pLeaf->szLeaf;
|
||||
|
||||
fts5SegIterNextPage(p, pIter);
|
||||
if( pIter->pLeaf==0 ) break;
|
||||
iOff = fts5LeafFirstRowidOff(pIter->pLeaf);
|
||||
if( iOff<4 || iOff>=n ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}else{
|
||||
iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
|
||||
pIter->iLeafOffset = iOff;
|
||||
fts5SegIterLoadNPos(p, pIter);
|
||||
if( iOff>0 ){
|
||||
u8 *a = pIter->pLeaf->p;
|
||||
int n = pIter->pLeaf->szLeaf;
|
||||
if( iOff<4 || iOff>=n ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}else{
|
||||
iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
|
||||
pIter->iLeafOffset = iOff;
|
||||
fts5SegIterLoadNPos(p, pIter);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3468,7 +3455,7 @@ static void fts5MultiIterNew(
|
||||
if( iLevel<0 ){
|
||||
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
|
||||
nSeg = pStruct->nSegment;
|
||||
nSeg += (p->pHash ? 1 : 0);
|
||||
nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH));
|
||||
}else{
|
||||
nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
|
||||
}
|
||||
@ -3489,7 +3476,7 @@ static void fts5MultiIterNew(
|
||||
if( p->rc==SQLITE_OK ){
|
||||
if( iLevel<0 ){
|
||||
Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
|
||||
if( p->pHash ){
|
||||
if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){
|
||||
/* Add a segment iterator for the current contents of the hash table. */
|
||||
Fts5SegIter *pIter = &pNew->aSeg[iIter++];
|
||||
fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
|
||||
@ -4244,7 +4231,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
|
||||
fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
|
||||
fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
|
||||
fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
|
||||
fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]);
|
||||
fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]);
|
||||
if( p->rc==SQLITE_OK ){
|
||||
/* Set the szLeaf field */
|
||||
fts5PutU16(&buf.p[2], (u16)buf.n);
|
||||
@ -4565,6 +4552,368 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
** Execute the SQL statement:
|
||||
**
|
||||
** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
|
||||
**
|
||||
** This is used when a secure-delete operation removes the last term
|
||||
** from a segment leaf page. In that case the %_idx entry is removed
|
||||
** too. This is done to ensure that if all instances of a token are
|
||||
** removed from an fts5 database in secure-delete mode, no trace of
|
||||
** the token itself remains in the database.
|
||||
*/
|
||||
static void fts5SecureDeleteIdxEntry(
|
||||
Fts5Index *p, /* FTS5 backend object */
|
||||
int iSegid, /* Id of segment to delete entry for */
|
||||
int iPgno /* Page number within segment */
|
||||
){
|
||||
if( iPgno!=1 ){
|
||||
assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE );
|
||||
if( p->pDeleteFromIdx==0 ){
|
||||
fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf(
|
||||
"DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
|
||||
p->pConfig->zDb, p->pConfig->zName
|
||||
));
|
||||
}
|
||||
if( p->rc==SQLITE_OK ){
|
||||
sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid);
|
||||
sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno);
|
||||
sqlite3_step(p->pDeleteFromIdx);
|
||||
p->rc = sqlite3_reset(p->pDeleteFromIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
** This is called when a secure-delete operation removes a position-list
|
||||
** that overflows onto segment page iPgno of segment pSeg. This function
|
||||
** rewrites node iPgno, and possibly one or more of its right-hand peers,
|
||||
** to remove this portion of the position list.
|
||||
**
|
||||
** Output variable (*pbLastInDoclist) is set to true if the position-list
|
||||
** removed is followed by a new term or the end-of-segment, or false if
|
||||
** it is followed by another rowid/position list.
|
||||
*/
|
||||
static void fts5SecureDeleteOverflow(
|
||||
Fts5Index *p,
|
||||
Fts5StructureSegment *pSeg,
|
||||
int iPgno,
|
||||
int *pbLastInDoclist
|
||||
){
|
||||
int pgno;
|
||||
Fts5Data *pLeaf = 0;
|
||||
assert( iPgno!=1 );
|
||||
|
||||
*pbLastInDoclist = 1;
|
||||
for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){
|
||||
i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno);
|
||||
int iNext = 0;
|
||||
u8 *aPg = 0;
|
||||
|
||||
pLeaf = fts5DataRead(p, iRowid);
|
||||
if( pLeaf==0 ) break;
|
||||
aPg = pLeaf->p;
|
||||
|
||||
iNext = fts5GetU16(&aPg[0]);
|
||||
if( iNext!=0 ){
|
||||
*pbLastInDoclist = 0;
|
||||
}
|
||||
if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){
|
||||
fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext);
|
||||
}
|
||||
|
||||
if( iNext==0 ){
|
||||
/* The page contains no terms or rowids. Replace it with an empty
|
||||
** page and move on to the right-hand peer. */
|
||||
const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
|
||||
fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
|
||||
fts5DataRelease(pLeaf);
|
||||
pLeaf = 0;
|
||||
}else{
|
||||
int nShift = iNext - 4;
|
||||
int nPg;
|
||||
|
||||
int nIdx = 0;
|
||||
u8 *aIdx = 0;
|
||||
|
||||
/* Unless the current page footer is 0 bytes in size (in which case
|
||||
** the new page footer will be as well), allocate and populate a
|
||||
** buffer containing the new page footer. Set stack variables aIdx
|
||||
** and nIdx accordingly. */
|
||||
if( pLeaf->nn>pLeaf->szLeaf ){
|
||||
int iFirst = 0;
|
||||
int i1 = pLeaf->szLeaf;
|
||||
int i2 = 0;
|
||||
|
||||
aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
|
||||
if( aIdx==0 ) break;
|
||||
i1 += fts5GetVarint32(&aPg[i1], iFirst);
|
||||
i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
|
||||
if( i1<pLeaf->nn ){
|
||||
memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
|
||||
i2 += (pLeaf->nn-i1);
|
||||
}
|
||||
nIdx = i2;
|
||||
}
|
||||
|
||||
/* Modify the contents of buffer aPg[]. Set nPg to the new size
|
||||
** in bytes. The new page is always smaller than the old. */
|
||||
nPg = pLeaf->szLeaf - nShift;
|
||||
memmove(&aPg[4], &aPg[4+nShift], nPg-4);
|
||||
fts5PutU16(&aPg[2], nPg);
|
||||
if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
|
||||
if( nIdx>0 ){
|
||||
memcpy(&aPg[nPg], aIdx, nIdx);
|
||||
nPg += nIdx;
|
||||
}
|
||||
sqlite3_free(aIdx);
|
||||
|
||||
/* Write the new page to disk and exit the loop */
|
||||
assert( nPg>4 || fts5GetU16(aPg)==0 );
|
||||
fts5DataWrite(p, iRowid, aPg, nPg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
fts5DataRelease(pLeaf);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
** This is called as part of flushing a delete to disk in 'secure-delete'
|
||||
** mode. It edits the segments within the database described by argument
|
||||
** pStruct to remove the entries for term zTerm, rowid iRowid.
|
||||
*/
|
||||
static void fts5FlushSecureDelete(
|
||||
Fts5Index *p,
|
||||
Fts5Structure *pStruct,
|
||||
const char *zTerm,
|
||||
i64 iRowid
|
||||
){
|
||||
const int f = FTS5INDEX_QUERY_SKIPHASH;
|
||||
int nTerm = strlen(zTerm);
|
||||
Fts5Iter *pIter = 0; /* Used to find term instance */
|
||||
|
||||
fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
|
||||
if( fts5MultiIterEof(p, pIter)==0 ){
|
||||
i64 iThis = fts5MultiIterRowid(pIter);
|
||||
if( iThis<iRowid ){
|
||||
fts5MultiIterNextFrom(p, pIter, iRowid);
|
||||
}
|
||||
|
||||
if( p->rc==SQLITE_OK
|
||||
&& fts5MultiIterEof(p, pIter)==0
|
||||
&& iRowid==fts5MultiIterRowid(pIter)
|
||||
){
|
||||
Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
|
||||
int iSegid = pSeg->pSeg->iSegid;
|
||||
u8 *aPg = pSeg->pLeaf->p;
|
||||
int nPg = pSeg->pLeaf->nn;
|
||||
int iPgIdx = pSeg->pLeaf->szLeaf;
|
||||
|
||||
u64 iDelta = 0;
|
||||
u64 iNextDelta = 0;
|
||||
int iNextOff = 0;
|
||||
int iOff = 0;
|
||||
int nMove = 0;
|
||||
int nIdx = 0;
|
||||
|
||||
u8 *aIdx = 0;
|
||||
|
||||
nIdx = nPg-iPgIdx;
|
||||
aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
|
||||
if( aIdx ){
|
||||
int bLastInDoclist = 0;
|
||||
int iIdx = 0;
|
||||
int iStart = 0;
|
||||
int iKeyOff = 0;
|
||||
int iPrevKeyOff = 0;
|
||||
int nShift = 0;
|
||||
int iDelKeyOff = 0; /* Offset of deleted key, if any */
|
||||
memcpy(aIdx, &aPg[iPgIdx], nIdx);
|
||||
|
||||
/* At this point segment iterator pSeg points to the entry
|
||||
** this function should remove from the b-tree segment.
|
||||
**
|
||||
** More specifically, pSeg->iLeafOffset is the offset of the
|
||||
** first byte in the position-list for the entry to remove.
|
||||
** Immediately before this comes two varints that will also
|
||||
** need to be removed:
|
||||
**
|
||||
** + the rowid or delta rowid value for the entry, and
|
||||
** + the size of the position list in bytes.
|
||||
*/
|
||||
{
|
||||
int iSOP;
|
||||
int nPos = 0;
|
||||
if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
|
||||
iStart = pSeg->iTermLeafOffset;
|
||||
}else{
|
||||
iStart = fts5GetU16(&aPg[0]);
|
||||
}
|
||||
|
||||
iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
|
||||
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
|
||||
assert_nc( iSOP<=pSeg->iLeafOffset );
|
||||
while( iSOP<pSeg->iLeafOffset ){
|
||||
iStart = iSOP + (nPos/2);
|
||||
iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
|
||||
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
|
||||
}
|
||||
assert_nc( iSOP==pSeg->iLeafOffset );
|
||||
}
|
||||
|
||||
iOff = iStart;
|
||||
iNextOff = pSeg->iLeafOffset + pSeg->nPos;
|
||||
if( iNextOff>=iPgIdx ){
|
||||
int pgno = pSeg->iLeafPgno+1;
|
||||
fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
|
||||
iNextOff = iPgIdx;
|
||||
}else{
|
||||
/* Set bLastInDoclist to true if the entry being removed is the last
|
||||
** in its doclist. */
|
||||
for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
|
||||
u32 iVal = 0;
|
||||
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
|
||||
iKeyOff += iVal;
|
||||
if( iKeyOff==iNextOff ){
|
||||
bLastInDoclist = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
|
||||
fts5PutU16(&aPg[0], 0);
|
||||
}
|
||||
|
||||
if( bLastInDoclist==0 ){
|
||||
if( iNextOff!=iPgIdx ){
|
||||
iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
|
||||
iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
|
||||
}
|
||||
}else if(
|
||||
iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno
|
||||
){
|
||||
/* The entry being removed was the only position list in its
|
||||
** doclist. Therefore the term needs to be removed as well. */
|
||||
int iKey = 0;
|
||||
for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
|
||||
u32 iVal = 0;
|
||||
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
|
||||
if( (iKeyOff+iVal)>iStart ) break;
|
||||
iKeyOff += iVal;
|
||||
}
|
||||
|
||||
iDelKeyOff = iOff = iKeyOff;
|
||||
if( iNextOff!=iPgIdx ){
|
||||
int nPrefix = 0;
|
||||
int nSuffix = 0;
|
||||
int nPrefix2 = 0;
|
||||
int nSuffix2 = 0;
|
||||
|
||||
iDelKeyOff = iNextOff;
|
||||
iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
|
||||
iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
|
||||
|
||||
if( iKey!=1 ){
|
||||
iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
|
||||
}
|
||||
iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
|
||||
|
||||
nPrefix = MIN(nPrefix, nPrefix2);
|
||||
nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
|
||||
|
||||
if( iKey!=1 ){
|
||||
iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
|
||||
}
|
||||
iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
|
||||
if( nPrefix2>nPrefix ){
|
||||
memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix);
|
||||
iOff += (nPrefix2-nPrefix);
|
||||
}
|
||||
memcpy(&aPg[iOff], &aPg[iNextOff], nSuffix2);
|
||||
iOff += nSuffix2;
|
||||
iNextOff += nSuffix2;
|
||||
}
|
||||
}else if( iStart==4 ){
|
||||
assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
|
||||
/* The entry being removed may be the only position list in
|
||||
** its doclist. */
|
||||
int iPgno = pSeg->iLeafPgno-1;
|
||||
|
||||
for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
|
||||
Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
|
||||
int bEmpty = (pPg && pPg->nn==4);
|
||||
fts5DataRelease(pPg);
|
||||
if( bEmpty==0 ) break;
|
||||
}
|
||||
|
||||
if( iPgno==pSeg->iTermLeafPgno ){
|
||||
i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
|
||||
Fts5Data *pTerm = fts5DataRead(p, iId);
|
||||
if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
|
||||
u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
|
||||
int nTermIdx = pTerm->nn - pTerm->szLeaf;
|
||||
int iTermIdx = 0;
|
||||
int iTermOff = 0;
|
||||
|
||||
while( 1 ){
|
||||
u32 iVal = 0;
|
||||
int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
|
||||
iTermOff += iVal;
|
||||
if( (iTermIdx+nByte)>=nTermIdx ) break;
|
||||
iTermIdx += nByte;
|
||||
}
|
||||
nTermIdx = iTermIdx;
|
||||
|
||||
memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
|
||||
fts5PutU16(&pTerm->p[2], iTermOff);
|
||||
|
||||
fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
|
||||
if( nTermIdx==0 ){
|
||||
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
|
||||
}
|
||||
}
|
||||
fts5DataRelease(pTerm);
|
||||
}
|
||||
}
|
||||
|
||||
nMove = nPg - iNextOff;
|
||||
memmove(&aPg[iOff], &aPg[iNextOff], nMove);
|
||||
iPgIdx -= (iNextOff - iOff);
|
||||
nPg = iPgIdx;
|
||||
fts5PutU16(&aPg[2], iPgIdx);
|
||||
|
||||
nShift = iNextOff - iOff;
|
||||
for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
|
||||
u32 iVal = 0;
|
||||
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
|
||||
iKeyOff += iVal;
|
||||
if( iKeyOff!=iDelKeyOff ){
|
||||
if( iKeyOff>iOff ){
|
||||
iKeyOff -= nShift;
|
||||
nShift = 0;
|
||||
}
|
||||
nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
|
||||
iPrevKeyOff = iKeyOff;
|
||||
}
|
||||
}
|
||||
|
||||
if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
|
||||
fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
|
||||
}
|
||||
|
||||
assert( nPg>4 || fts5GetU16(aPg)==0 );
|
||||
fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg);
|
||||
sqlite3_free(aIdx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fts5MultiIterFree(pIter);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
** Flush the contents of in-memory hash table iHash to a new level-0
|
||||
** segment on disk. Also update the corresponding structure record.
|
||||
@ -4587,6 +4936,7 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
if( iSegid ){
|
||||
const int pgsz = p->pConfig->pgsz;
|
||||
int eDetail = p->pConfig->eDetail;
|
||||
int bSecureDelete = p->pConfig->bSecureDelete;
|
||||
Fts5StructureSegment *pSeg; /* New segment within pStruct */
|
||||
Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */
|
||||
Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */
|
||||
@ -4609,40 +4959,64 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
}
|
||||
while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
|
||||
const char *zTerm; /* Buffer containing term */
|
||||
int nTerm; /* Size of zTerm in bytes */
|
||||
const u8 *pDoclist; /* Pointer to doclist for this term */
|
||||
int nDoclist; /* Size of doclist in bytes */
|
||||
|
||||
/* Write the term for this entry to disk. */
|
||||
/* Get the term and doclist for this entry. */
|
||||
sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
|
||||
fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm);
|
||||
if( p->rc!=SQLITE_OK ) break;
|
||||
nTerm = (int)strlen(zTerm);
|
||||
if( bSecureDelete==0 ){
|
||||
fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
|
||||
if( p->rc!=SQLITE_OK ) break;
|
||||
assert( writer.bFirstRowidInPage==0 );
|
||||
}
|
||||
|
||||
assert( writer.bFirstRowidInPage==0 );
|
||||
if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
|
||||
if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
|
||||
/* The entire doclist will fit on the current leaf. */
|
||||
fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
|
||||
}else{
|
||||
int bTermWritten = !bSecureDelete;
|
||||
i64 iRowid = 0;
|
||||
u64 iDelta = 0;
|
||||
i64 iPrev = 0;
|
||||
int iOff = 0;
|
||||
|
||||
/* The entire doclist will not fit on this leaf. The following
|
||||
** loop iterates through the poslists that make up the current
|
||||
** doclist. */
|
||||
while( p->rc==SQLITE_OK && iOff<nDoclist ){
|
||||
u64 iDelta = 0;
|
||||
iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
|
||||
iRowid += iDelta;
|
||||
|
||||
/* If in secure delete mode, and if this entry in the poslist is
|
||||
** in fact a delete, then edit the existing segments directly
|
||||
** using fts5FlushSecureDelete(). */
|
||||
if( bSecureDelete && (pDoclist[iOff] & 0x01) ){
|
||||
fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
|
||||
if( pDoclist[iOff]==0x01 ){
|
||||
iOff++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if( bTermWritten==0 ){
|
||||
fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
|
||||
bTermWritten = 1;
|
||||
assert( writer.bFirstRowidInPage==0 );
|
||||
}
|
||||
|
||||
if( writer.bFirstRowidInPage ){
|
||||
fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */
|
||||
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
|
||||
writer.bFirstRowidInPage = 0;
|
||||
fts5WriteDlidxAppend(p, &writer, iRowid);
|
||||
if( p->rc!=SQLITE_OK ) break;
|
||||
}else{
|
||||
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
|
||||
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
|
||||
}
|
||||
if( p->rc!=SQLITE_OK ) break;
|
||||
assert( pBuf->n<=pBuf->nSpace );
|
||||
iPrev = iRowid;
|
||||
|
||||
if( eDetail==FTS5_DETAIL_NONE ){
|
||||
if( iOff<nDoclist && pDoclist[iOff]==0 ){
|
||||
@ -4701,20 +5075,23 @@ static void fts5FlushOneHash(Fts5Index *p){
|
||||
sqlite3Fts5HashClear(pHash);
|
||||
fts5WriteFinish(p, &writer, &pgnoLast);
|
||||
|
||||
/* Update the Fts5Structure. It is written back to the database by the
|
||||
** fts5StructureRelease() call below. */
|
||||
if( pStruct->nLevel==0 ){
|
||||
fts5StructureAddLevel(&p->rc, &pStruct);
|
||||
assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
|
||||
if( pgnoLast>0 ){
|
||||
/* Update the Fts5Structure. It is written back to the database by the
|
||||
** fts5StructureRelease() call below. */
|
||||
if( pStruct->nLevel==0 ){
|
||||
fts5StructureAddLevel(&p->rc, &pStruct);
|
||||
}
|
||||
fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
|
||||
if( p->rc==SQLITE_OK ){
|
||||
pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
|
||||
pSeg->iSegid = iSegid;
|
||||
pSeg->pgnoFirst = 1;
|
||||
pSeg->pgnoLast = pgnoLast;
|
||||
pStruct->nSegment++;
|
||||
}
|
||||
fts5StructurePromote(p, 0, pStruct);
|
||||
}
|
||||
fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
|
||||
if( p->rc==SQLITE_OK ){
|
||||
pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
|
||||
pSeg->iSegid = iSegid;
|
||||
pSeg->pgnoFirst = 1;
|
||||
pSeg->pgnoLast = pgnoLast;
|
||||
pStruct->nSegment++;
|
||||
}
|
||||
fts5StructurePromote(p, 0, pStruct);
|
||||
}
|
||||
|
||||
fts5IndexAutomerge(p, &pStruct, pgnoLast);
|
||||
@ -5455,6 +5832,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
|
||||
sqlite3_finalize(p->pIdxDeleter);
|
||||
sqlite3_finalize(p->pIdxSelect);
|
||||
sqlite3_finalize(p->pDataVersion);
|
||||
sqlite3_finalize(p->pDeleteFromIdx);
|
||||
sqlite3Fts5HashFree(p->pHash);
|
||||
sqlite3_free(p->zDataTbl);
|
||||
sqlite3_free(p);
|
||||
@ -6085,6 +6463,7 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
Fts5StructureSegment *pSeg /* Segment to check internal consistency */
|
||||
){
|
||||
Fts5Config *pConfig = p->pConfig;
|
||||
int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE);
|
||||
sqlite3_stmt *pStmt = 0;
|
||||
int rc2;
|
||||
int iIdxPrevLeaf = pSeg->pgnoFirst-1;
|
||||
@ -6120,7 +6499,19 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
** is also a rowid pointer within the leaf page header, it points to a
|
||||
** location before the term. */
|
||||
if( pLeaf->nn<=pLeaf->szLeaf ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
|
||||
if( nIdxTerm==0
|
||||
&& pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE
|
||||
&& pLeaf->nn==pLeaf->szLeaf
|
||||
&& pLeaf->nn==4
|
||||
){
|
||||
/* special case - the very first page in a segment keeps its %_idx
|
||||
** entry even if all the terms are removed from it by secure-delete
|
||||
** operations. */
|
||||
}else{
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
|
||||
}else{
|
||||
int iOff; /* Offset of first term on leaf */
|
||||
int iRowidOff; /* Offset of first rowid on leaf */
|
||||
@ -6184,9 +6575,12 @@ static void fts5IndexIntegrityCheckSegment(
|
||||
ASSERT_SZLEAF_OK(pLeaf);
|
||||
if( iRowidOff>=pLeaf->szLeaf ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}else{
|
||||
}else if( bSecureDelete==0 || iRowidOff>0 ){
|
||||
i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
|
||||
fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
|
||||
if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
|
||||
if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
|
||||
p->rc = FTS5_CORRUPT;
|
||||
}
|
||||
}
|
||||
fts5DataRelease(pLeaf);
|
||||
}
|
||||
|
||||
@ -1623,6 +1623,8 @@ static int fts5UpdateMethod(
|
||||
Fts5Config *pConfig = pTab->p.pConfig;
|
||||
int eType0; /* value_type() of apVal[0] */
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
int bUpdateOrDelete = 0;
|
||||
|
||||
|
||||
/* A transaction must be open when this is called. */
|
||||
assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
|
||||
@ -1633,6 +1635,11 @@ static int fts5UpdateMethod(
|
||||
|| sqlite3_value_type(apVal[0])==SQLITE_NULL
|
||||
);
|
||||
assert( pTab->p.pConfig->pzErrmsg==0 );
|
||||
if( pConfig->pgsz==0 ){
|
||||
rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
}
|
||||
|
||||
pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
|
||||
|
||||
/* Put any active cursors into REQUIRE_SEEK state. */
|
||||
@ -1685,6 +1692,7 @@ static int fts5UpdateMethod(
|
||||
else if( nArg==1 ){
|
||||
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
|
||||
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
|
||||
bUpdateOrDelete = 1;
|
||||
}
|
||||
|
||||
/* INSERT or UPDATE */
|
||||
@ -1700,6 +1708,7 @@ static int fts5UpdateMethod(
|
||||
if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
|
||||
i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */
|
||||
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
|
||||
bUpdateOrDelete = 1;
|
||||
}
|
||||
fts5StorageInsert(&rc, pTab, apVal, pRowid);
|
||||
}
|
||||
@ -1728,10 +1737,24 @@ static int fts5UpdateMethod(
|
||||
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
|
||||
fts5StorageInsert(&rc, pTab, apVal, pRowid);
|
||||
}
|
||||
bUpdateOrDelete = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK
|
||||
&& bUpdateOrDelete
|
||||
&& pConfig->bSecureDelete
|
||||
&& pConfig->iVersion==FTS5_CURRENT_VERSION
|
||||
){
|
||||
rc = sqlite3Fts5StorageConfigValue(
|
||||
pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE
|
||||
);
|
||||
if( rc==SQLITE_OK ){
|
||||
pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE;
|
||||
}
|
||||
}
|
||||
|
||||
pTab->p.pConfig->pzErrmsg = 0;
|
||||
return rc;
|
||||
}
|
||||
@ -2591,6 +2614,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
|
||||
UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */
|
||||
fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
|
||||
fts5TripCursors(pTab);
|
||||
pTab->p.pConfig->pgsz = 0;
|
||||
return sqlite3Fts5StorageRollback(pTab->pStorage);
|
||||
}
|
||||
|
||||
|
||||
@ -594,6 +594,10 @@ proc nearset_rc {aCol args} {
|
||||
list
|
||||
}
|
||||
|
||||
proc dump {tname} {
|
||||
execsql_pp "SELECT * FROM ${tname}_idx"
|
||||
execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Code for a simple Tcl tokenizer that supports synonyms at query time.
|
||||
|
||||
278
ext/fts5/test/fts5secure.test
Normal file
278
ext/fts5/test/fts5secure.test
Normal file
@ -0,0 +1,278 @@
|
||||
# 2023 Feb 17
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
ifcapable !fts5 { finish_test ; return }
|
||||
set ::testprefix fts5secure
|
||||
|
||||
proc dump {tname} {
|
||||
execsql_pp "SELECT * FROM ${tname}_idx"
|
||||
execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
|
||||
}
|
||||
|
||||
|
||||
do_execsql_test 0.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance');
|
||||
INSERT INTO t1(rowid, ab) VALUES
|
||||
(0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def');
|
||||
}
|
||||
|
||||
do_execsql_test 0.1 {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
do_execsql_test 0.2 {
|
||||
DELETE FROM t1 WHERE rowid=2;
|
||||
}
|
||||
|
||||
do_execsql_test 0.3 {
|
||||
SELECT count(*) FROM t1_data
|
||||
} 3
|
||||
|
||||
do_execsql_test 0.4 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 0.5 {
|
||||
DELETE FROM t1 WHERE rowid=3;
|
||||
}
|
||||
|
||||
do_execsql_test 0.6 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 0.7 {
|
||||
DELETE FROM t1 WHERE rowid=0;
|
||||
}
|
||||
|
||||
do_execsql_test 0.8 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
#----------------------------------
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE t2 USING fts5(ab);
|
||||
INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value');
|
||||
INSERT INTO t2(t2, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
#execsql_pp { SELECT id, quote(block) FROM t1_data }
|
||||
#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx }
|
||||
|
||||
do_execsql_test 1.1 {
|
||||
DELETE FROM t2 WHERE rowid = 5;
|
||||
}
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
INSERT INTO t2(t2) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
DELETE FROM t2 WHERE rowid = 6;
|
||||
}
|
||||
|
||||
do_execsql_test 1.4 {
|
||||
INSERT INTO t2(t2) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 1.5 {
|
||||
SELECT * FROM t2('value');
|
||||
SELECT * FROM t2('v*');
|
||||
}
|
||||
|
||||
do_execsql_test 1.6 {
|
||||
SELECT * FROM t2('value') ORDER BY rowid DESC;
|
||||
SELECT * FROM t2('v*') ORDER BY rowid DESC;
|
||||
}
|
||||
execsql_pp {
|
||||
SELECT id, quote(block) FROM t2_data;
|
||||
}
|
||||
|
||||
#----------------------------------
|
||||
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE ft USING fts5(ab);
|
||||
CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance');
|
||||
INSERT INTO ft(rowid, ab) VALUES
|
||||
(1, 'one'),
|
||||
(2, 'two'),
|
||||
(3, 'three'),
|
||||
(4, 'four'),
|
||||
(5, 'one one'),
|
||||
(6, 'one two'),
|
||||
(7, 'one three'),
|
||||
(8, 'one four'),
|
||||
(9, 'two one'),
|
||||
(10, 'two two'),
|
||||
(11, 'two three'),
|
||||
(12, 'two four'),
|
||||
(13, 'three one'),
|
||||
(14, 'three two'),
|
||||
(15, 'three three'),
|
||||
(16, 'three four');
|
||||
}
|
||||
|
||||
do_execsql_test 2.1 {
|
||||
SELECT count(*) FROM ft_data;
|
||||
} {3}
|
||||
|
||||
do_execsql_test 2.2 {
|
||||
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
do_execsql_test 2.3 {
|
||||
DELETE FROM ft WHERE rowid=9;
|
||||
}
|
||||
|
||||
do_execsql_test 2.4 {
|
||||
INSERT INTO ft(ft) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 2.5 {
|
||||
DELETE FROM ft WHERE ab LIKE '%two%'
|
||||
}
|
||||
|
||||
do_execsql_test 2.6 {
|
||||
INSERT INTO ft(ft) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 2.7 {
|
||||
SELECT count(*) FROM ft_data;
|
||||
} {3}
|
||||
|
||||
#----------------------------------
|
||||
reset_db
|
||||
|
||||
set ::vocab {
|
||||
one two three four five six seven eight nine ten
|
||||
eleven twelve thirteen fourteen fifteen sixteen
|
||||
seventeen eighteen nineteen twenty
|
||||
}
|
||||
proc rnddoc {} {
|
||||
set nVocab [llength $::vocab]
|
||||
set ret [list]
|
||||
for {set ii 0} {$ii < 8} {incr ii} {
|
||||
lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]]
|
||||
}
|
||||
set ret
|
||||
}
|
||||
|
||||
proc contains {list val} {
|
||||
expr {[lsearch $list $val]>=0}
|
||||
}
|
||||
|
||||
foreach {tn pgsz} {
|
||||
2 64
|
||||
1 1000
|
||||
} {
|
||||
reset_db
|
||||
db function rnddoc rnddoc
|
||||
db function contains contains
|
||||
|
||||
expr srand(1)
|
||||
|
||||
do_execsql_test 3.$tn.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(x);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
|
||||
WITH s(i) AS (
|
||||
VALUES(1) UNION SELECT i+1 FROM s WHERE i<20
|
||||
)
|
||||
INSERT INTO t1 SELECT rnddoc() FROM s;
|
||||
}
|
||||
|
||||
do_execsql_test 3.$tn.1 {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
foreach {rowid} {
|
||||
6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8
|
||||
} {
|
||||
|
||||
do_execsql_test 3.$tn.2.$rowid {
|
||||
DELETE FROM t1 WHERE rowid=$rowid;
|
||||
}
|
||||
do_execsql_test 3.$tn.2.$rowid.ic {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
foreach v $::vocab {
|
||||
do_execsql_test 3.$tn.2.$rowid.q.$v {
|
||||
SELECT rowid FROM t1($v)
|
||||
} [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}]
|
||||
|
||||
do_execsql_test 3.$tn.2.$rowid.q.$v.DESC {
|
||||
SELECT rowid FROM t1($v) ORDER BY 1 DESC
|
||||
} [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_execsql_test 3.3 {
|
||||
INSERT INTO t1(x) VALUES('optimize');
|
||||
INSERT INTO t1(t1) VALUES('optimize');
|
||||
SELECT count(*) FROM t1_data;
|
||||
} {3}
|
||||
|
||||
#----------------------------------
|
||||
reset_db
|
||||
do_execsql_test 4.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(x);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
set L1 [string repeat abcdefghij 10]
|
||||
set L2 [string repeat 1234567890 10]
|
||||
|
||||
do_execsql_test 4.1 {
|
||||
INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2);
|
||||
}
|
||||
do_execsql_test 4.2 {
|
||||
DELETE FROM t1 WHERE rowid=1
|
||||
}
|
||||
do_execsql_test 4.3 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
#----------------------------------
|
||||
reset_db
|
||||
do_execsql_test 5.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(x);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
set doc "aa [string repeat {abc } 60]"
|
||||
|
||||
do_execsql_test 5.1 {
|
||||
BEGIN;
|
||||
INSERT INTO t1 VALUES($doc);
|
||||
INSERT INTO t1 VALUES('aa abc');
|
||||
COMMIT;
|
||||
}
|
||||
|
||||
do_execsql_test 5.2 {
|
||||
DELETE FROM t1 WHERE rowid = 1;
|
||||
}
|
||||
|
||||
do_execsql_test 5.3 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
|
||||
do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
87
ext/fts5/test/fts5secure2.test
Normal file
87
ext/fts5/test/fts5secure2.test
Normal file
@ -0,0 +1,87 @@
|
||||
# 2023 Feb 17
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
ifcapable !fts5 { finish_test ; return }
|
||||
set ::testprefix fts5secure2
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE ft USING fts5(col);
|
||||
INSERT INTO ft VALUES('data for the table');
|
||||
INSERT INTO ft VALUES('more of the same');
|
||||
INSERT INTO ft VALUES('and extra data');
|
||||
}
|
||||
|
||||
do_execsql_test 1.1 {
|
||||
SELECT * FROM ft_config
|
||||
} {version 4}
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
|
||||
SELECT * FROM ft_config;
|
||||
} {secure-delete 1 version 4}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
|
||||
SELECT * FROM ft_config;
|
||||
} {secure-delete 1 version 4}
|
||||
|
||||
do_execsql_test 1.4 {
|
||||
DELETE FROM ft WHERE rowid=2;
|
||||
SELECT * FROM ft_config;
|
||||
} {secure-delete 1 version 5}
|
||||
|
||||
do_execsql_test 1.5 {
|
||||
SELECT rowid, col FROM ft('data');
|
||||
} {1 {data for the table} 3 {and extra data}}
|
||||
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
|
||||
do_execsql_test 1.6 {
|
||||
SELECT rowid, col FROM ft('data');
|
||||
} {1 {data for the table} 3 {and extra data}}
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
|
||||
reset_db
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE ft USING fts5(col);
|
||||
INSERT INTO ft VALUES('one zero one one zero');
|
||||
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
do_execsql_test 2.1 {
|
||||
SELECT count(*) FROM ft_data WHERE block=X'00000004';
|
||||
} {0}
|
||||
|
||||
do_execsql_test 2.2 {
|
||||
UPDATE ft SET col = 'zero one zero zero one' WHERE rowid=1;
|
||||
}
|
||||
|
||||
do_execsql_test 2.3 {
|
||||
SELECT count(*) FROM ft_data WHERE block=X'00000004';
|
||||
} {1}
|
||||
|
||||
do_execsql_test 2.4 {
|
||||
INSERT INTO ft VALUES('one zero zero one');
|
||||
DELETE FROM ft WHERE rowid=1;
|
||||
}
|
||||
|
||||
do_execsql_test 2.5 {
|
||||
SELECT count(*) FROM ft_data WHERE block=X'00000004';
|
||||
} {2}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
|
||||
166
ext/fts5/test/fts5secure3.test
Normal file
166
ext/fts5/test/fts5secure3.test
Normal file
@ -0,0 +1,166 @@
|
||||
# 2023 Feb 17
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
# TESTRUNNER: slow
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
ifcapable !fts5 { finish_test ; return }
|
||||
set ::testprefix fts5secure3
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE ft USING fts5(col);
|
||||
INSERT INTO ft VALUES('data for the table');
|
||||
INSERT INTO ft VALUES('more of the same');
|
||||
INSERT INTO ft VALUES('and extra data');
|
||||
|
||||
INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
do_execsql_test 1.1 {
|
||||
BEGIN;
|
||||
INSERT INTO ft(rowid, col) VALUES(0, 'the next data');
|
||||
DELETE FROM ft WHERE rowid=1;
|
||||
DELETE FROM ft WHERE rowid=2;
|
||||
INSERT INTO ft(rowid, col) VALUES(6, 'with some more of the same data');
|
||||
COMMIT;
|
||||
}
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
INSERT INTO ft(ft) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
|
||||
reset_db
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(x);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
BEGIN;
|
||||
INSERT INTO t1 VALUES('the start');
|
||||
}
|
||||
do_test 2.1 {
|
||||
for {set i 0} {$i < 1000} {incr i} {
|
||||
execsql { INSERT INTO t1 VALUES('the ' || hex(randomblob(3))) }
|
||||
}
|
||||
execsql {
|
||||
INSERT INTO t1 VALUES('the end');
|
||||
COMMIT;
|
||||
}
|
||||
} {}
|
||||
|
||||
do_execsql_test 2.2 {
|
||||
DELETE FROM t1 WHERE rowid BETWEEN 2 AND 1000;
|
||||
}
|
||||
|
||||
do_execsql_test 2.3 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 2.6 {
|
||||
INSERT INTO t1(rowid, x) VALUES(500, 'middle');
|
||||
INSERT INTO t1(rowid, x) VALUES(501, 'value');
|
||||
SELECT * FROM t1('the middle');
|
||||
}
|
||||
|
||||
do_execsql_test 2.7 {
|
||||
INSERT INTO t1(t1) VALUES('optimize');
|
||||
}
|
||||
|
||||
do_execsql_test 2.8 {
|
||||
SELECT count(*) FROM t1_data
|
||||
} 4
|
||||
|
||||
#execsql_pp { SELECT id, quote(block), fts5_decode(id, block) FROM t1_data; }
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Tests with large/small rowid values.
|
||||
#
|
||||
|
||||
reset_db
|
||||
|
||||
expr srand(0)
|
||||
|
||||
set vocab {
|
||||
Popper Poppins Popsicle Porfirio Porrima Porsche
|
||||
Porter Portia Portland Portsmouth Portugal Portuguese
|
||||
Poseidon Post PostgreSQL Potemkin Potomac Potsdam
|
||||
Pottawatomie Potter Potts Pound Poussin Powell
|
||||
PowerPC PowerPoint Powers Powhatan Poznan Prada
|
||||
Prado Praetorian Prague Praia Prakrit Pratchett
|
||||
Pratt Pravda Praxiteles Preakness Precambrian Preminger
|
||||
Premyslid Prensa Prentice Pres Presbyterian Presbyterianism
|
||||
}
|
||||
proc newdoc {} {
|
||||
for {set i 0} {$i<8} {incr i} {
|
||||
lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]]
|
||||
}
|
||||
set ret
|
||||
}
|
||||
db func newdoc newdoc
|
||||
|
||||
do_execsql_test 3.0 {
|
||||
CREATE VIRTUAL TABLE fff USING fts5(y);
|
||||
INSERT INTO fff(fff, rank) VALUES('pgsz', 64);
|
||||
|
||||
WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
|
||||
INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
|
||||
|
||||
WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
|
||||
INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
|
||||
|
||||
WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 )
|
||||
INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s;
|
||||
|
||||
INSERT INTO fff(fff, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
proc lshuffle {in} {
|
||||
set out [list]
|
||||
while {[llength $in]>0} {
|
||||
set idx [expr int(abs(rand()) * [llength $in])]
|
||||
lappend out [lindex $in $idx]
|
||||
set in [lreplace $in $idx $idx]
|
||||
}
|
||||
set out
|
||||
}
|
||||
|
||||
#dump fff
|
||||
|
||||
set iTest 1
|
||||
foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] {
|
||||
#if {$iTest==1} { dump fff }
|
||||
#if {$iTest==1} { breakpoint }
|
||||
do_execsql_test 3.1.$iTest.$ii {
|
||||
DELETE FROM fff WHERE rowid=$ii;
|
||||
}
|
||||
#if {$iTest==1} { dump fff }
|
||||
if {($iTest % 20)==0} {
|
||||
do_execsql_test 3.1.$iTest.$ii.ic {
|
||||
INSERT INTO fff(fff) VALUES('integrity-check');
|
||||
}
|
||||
}
|
||||
#if {$iTest==1} { break }
|
||||
incr iTest
|
||||
}
|
||||
|
||||
#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC }
|
||||
#breakpoint
|
||||
#execsql_pp {
|
||||
# SELECT rowid FROM fff('post') ORDER BY rowid DESC
|
||||
#}
|
||||
#
|
||||
#dump fff
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
136
ext/fts5/test/fts5secure4.test
Normal file
136
ext/fts5/test/fts5secure4.test
Normal file
@ -0,0 +1,136 @@
|
||||
# 2023 April 14
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
return_if_no_fts5
|
||||
set ::testprefix fts5secure4
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Test using the 'delete' command to attempt to delete a token that
|
||||
# is not present in the index in secure-delete mode.
|
||||
#
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1);
|
||||
|
||||
CREATE TABLE x1(rowid INTEGER PRIMARY KEY, a, b);
|
||||
INSERT INTO x1 VALUES
|
||||
(1, 'hello world', 'today xyz'),
|
||||
(2, 'not the day', 'crunch crumble and chomp'),
|
||||
(3, 'one', 'two');
|
||||
INSERT INTO t1(t1) VALUES('rebuild');
|
||||
}
|
||||
|
||||
do_execsql_test 1.1 {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
|
||||
do_execsql_test 1.2 {
|
||||
INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 4, 'nosuchtoken', '');
|
||||
}
|
||||
|
||||
do_execsql_test 1.3 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 1.4 {
|
||||
INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'crunch', '');
|
||||
}
|
||||
|
||||
do_execsql_test 1.5 {
|
||||
INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 3, 'crunch', '');
|
||||
}
|
||||
|
||||
do_execsql_test 1.6 {
|
||||
INSERT INTO t1(t1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Test using secure-delete with detail=none or detail=col.
|
||||
#
|
||||
foreach {tn d} {1 full 2 none 3 column} {
|
||||
reset_db
|
||||
do_execsql_test 2.$tn.1 "
|
||||
CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2');
|
||||
INSERT INTO x1(x1, rank) VALUES('pgsz', 64);
|
||||
"
|
||||
|
||||
do_execsql_test 2.$tn.2 {
|
||||
BEGIN;
|
||||
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
|
||||
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
|
||||
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
|
||||
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
|
||||
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
|
||||
COMMIT;
|
||||
}
|
||||
|
||||
do_execsql_test 2.$tn.3 {
|
||||
DELETE FROM x1 WHERE rowid IN (2, 4, 6);
|
||||
INSERT INTO x1(x1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 2.$tn.4 {
|
||||
DELETE FROM x1 WHERE rowid IN (1, 3, 5);
|
||||
INSERT INTO x1(x1) VALUES('integrity-check');
|
||||
}
|
||||
|
||||
do_execsql_test 2.$tn.5 {
|
||||
WITH s(i) AS (
|
||||
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
|
||||
)
|
||||
INSERT INTO x1
|
||||
SELECT 'seems to be', 'used brew to', 'everything is working' FROM s
|
||||
UNION ALL
|
||||
SELECT 'used brew to', 'everything is working', 'seems to be' FROM s
|
||||
UNION ALL
|
||||
SELECT 'everything is working', 'seems to be', 'used brew to' FROM s
|
||||
UNION ALL
|
||||
SELECT 'abc', 'zzz', 'a b c d'
|
||||
UNION ALL
|
||||
SELECT 'z', 'z', 'z' FROM s
|
||||
}
|
||||
|
||||
do_test 2.$tn.6 {
|
||||
for {set i 300} {$i > 200} {incr i -1} {
|
||||
execsql {
|
||||
DELETE FROM x1 WHERE rowid=$i;
|
||||
INSERT INTO x1(x1) VALUES('integrity-check');
|
||||
}
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 2.$tn.7 {
|
||||
for {set i 1} {$i < 100} {incr i} {
|
||||
execsql {
|
||||
DELETE FROM x1 WHERE rowid=$i;
|
||||
INSERT INTO x1(x1) VALUES('integrity-check');
|
||||
}
|
||||
}
|
||||
} {}
|
||||
|
||||
do_test 2.$tn.8 {
|
||||
foreach i [db eval {SELECT rowid FROM x1}] {
|
||||
execsql {
|
||||
DELETE FROM x1 WHERE rowid=$i;
|
||||
INSERT INTO x1(x1) VALUES('integrity-check');
|
||||
}
|
||||
}
|
||||
} {}
|
||||
|
||||
do_execsql_test 2.$tn.9 {
|
||||
SELECT * FROM x1
|
||||
} {}
|
||||
}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
171
ext/fts5/test/fts5securefault.test
Normal file
171
ext/fts5/test/fts5securefault.test
Normal file
@ -0,0 +1,171 @@
|
||||
# 2023 April 14
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
# This file implements regression tests for SQLite library. The
|
||||
# focus of this script is testing the FTS5 module.
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
source $testdir/malloc_common.tcl
|
||||
set testprefix fts5securefault
|
||||
|
||||
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
|
||||
return_if_no_fts5
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
INSERT INTO t1(rowid, ab) VALUES
|
||||
(0, 'abc'), (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'def');
|
||||
}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 1.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid=2 }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
do_faultsim_test 1.2 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid IN(0, 1, 2, 3, 4) }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
set big [string repeat abcdefghij 5]
|
||||
set big2 [string repeat klmnopqrst 5]
|
||||
set doc "$big $big2"
|
||||
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
|
||||
WITH s(i) AS (
|
||||
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<4
|
||||
)
|
||||
INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s;
|
||||
}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 2.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid = 3 }
|
||||
execsql { DELETE FROM t1 WHERE rowid = 4 }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
set big [string repeat abcdefghij 5]
|
||||
set big2 [string repeat klmnopqrst 5]
|
||||
set doc "$big $big2"
|
||||
|
||||
do_execsql_test 3.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
|
||||
WITH s(i) AS (
|
||||
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<25
|
||||
)
|
||||
INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s;
|
||||
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
DELETE FROM t1 WHERE rowid BETWEEN 3 AND 23;
|
||||
}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 3.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid = 24 }
|
||||
execsql { DELETE FROM t1 WHERE rowid = 25 }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
set doc [string repeat "tok " 400]
|
||||
|
||||
do_execsql_test 4.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
|
||||
INSERT INTO t1(rowid, ab) VALUES(1, $doc), (2, $doc), (3, $doc);
|
||||
}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 4.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid = 2 }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
reset_db
|
||||
|
||||
set doc1 [string repeat "abc " 10]
|
||||
set doc2 [string repeat "def " 10]
|
||||
|
||||
do_test 5.0 {
|
||||
execsql {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(ab);
|
||||
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
|
||||
BEGIN;
|
||||
}
|
||||
for {set i 0} {$i < 50} {incr i} {
|
||||
execsql {
|
||||
INSERT INTO t1(rowid, ab) VALUES($i, 'abcdefg');
|
||||
}
|
||||
}
|
||||
execsql {
|
||||
INSERT INTO t1(rowid, ab) VALUES(105, 'def');
|
||||
COMMIT;
|
||||
}
|
||||
} {}
|
||||
faultsim_save_and_close
|
||||
|
||||
do_faultsim_test 5.1 -faults oom* -prep {
|
||||
faultsim_restore_and_reopen
|
||||
execsql {
|
||||
INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
|
||||
}
|
||||
} -body {
|
||||
execsql { DELETE FROM t1 WHERE rowid = 105 }
|
||||
} -test {
|
||||
faultsim_test_result {0 {}}
|
||||
}
|
||||
|
||||
|
||||
finish_test
|
||||
@ -38,20 +38,20 @@ do_execsql_test 1.3 {
|
||||
|
||||
sqlite3_db_config db DEFENSIVE 0
|
||||
do_execsql_test 1.4 {
|
||||
UPDATE t1_config set v=5 WHERE k='version';
|
||||
UPDATE t1_config set v=6 WHERE k='version';
|
||||
}
|
||||
|
||||
do_test 1.5 {
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
|
||||
} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}}
|
||||
|
||||
do_test 1.6 {
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { INSERT INTO t1 VALUES('x y z') }
|
||||
} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}}
|
||||
|
||||
do_test 1.7 {
|
||||
sqlite3_db_config db DEFENSIVE 0
|
||||
@ -59,7 +59,75 @@ do_test 1.7 {
|
||||
db close
|
||||
sqlite3 db test.db
|
||||
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
|
||||
} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
|
||||
} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
|
||||
|
||||
do_test 1.8 {
|
||||
sqlite3_db_config db DEFENSIVE 0
|
||||
execsql { INSERT INTO t1_config VALUES('version', 4) }
|
||||
execsql { INSERT INTO t1(t1, rank) VALUES('secure-delete', 1) }
|
||||
} {}
|
||||
|
||||
do_execsql_test 1.10 {
|
||||
SELECT * FROM t1_config
|
||||
} {secure-delete 1 version 4}
|
||||
|
||||
do_execsql_test 1.11 {
|
||||
INSERT INTO t1(rowid, one) VALUES(123, 'one two three');
|
||||
DELETE FROM t1 WHERE rowid=123;
|
||||
SELECT * FROM t1_config
|
||||
} {secure-delete 1 version 5}
|
||||
|
||||
do_execsql_test 1.11 {
|
||||
INSERT INTO t1(t1) VALUES('rebuild');
|
||||
SELECT * FROM t1_config
|
||||
} {secure-delete 1 version 4}
|
||||
|
||||
do_execsql_test 1.12 {
|
||||
SELECT * FROM t1_config
|
||||
} {secure-delete 1 version 4}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
reset_db
|
||||
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE xyz USING fts5(x);
|
||||
INSERT INTO xyz(rowid, x) VALUES
|
||||
(1, 'one document'),
|
||||
(2, 'two document'),
|
||||
(3, 'three document'),
|
||||
(4, 'four document'),
|
||||
(5, 'five document'),
|
||||
(6, 'six document');
|
||||
|
||||
INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1);
|
||||
SELECT v FROM xyz_config WHERE k='version';
|
||||
} {4}
|
||||
|
||||
do_execsql_test 2.1 {
|
||||
BEGIN;
|
||||
INSERT INTO xyz(rowid, x) VALUES(7, 'seven document');
|
||||
SAVEPOINT one;
|
||||
DELETE FROM xyz WHERE rowid = 4;
|
||||
}
|
||||
|
||||
do_execsql_test 2.2 {
|
||||
SELECT v FROM xyz_config WHERE k='version';
|
||||
} {5}
|
||||
|
||||
do_execsql_test 2.3 {
|
||||
ROLLBACK TO one;
|
||||
SELECT v FROM xyz_config WHERE k='version';
|
||||
} {4}
|
||||
|
||||
|
||||
do_execsql_test 2.4 {
|
||||
DELETE FROM xyz WHERE rowid = 3;
|
||||
COMMIT;
|
||||
SELECT v FROM xyz_config WHERE k='version';
|
||||
} {5}
|
||||
|
||||
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
|
||||
Reference in New Issue
Block a user