database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Fix some problems with using fts5 options 'secure-delete' and detail=none together.

Browse Source 
FossilOrigin-Name: 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed
This commit is contained in:
dan
2023-04-17 18:32:22 +00:00
parent 48824f23cb
commit 68c0da6f66
5 changed files with 410 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
ext/fts5/fts5_index.c
View File

@ -1954,7 +1954,7 @@ static void fts5SegIterNext_None(
iOff = pIter->iLeafOffset;
/* Next entry is on the next page */
if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
fts5SegIterNextPage(p, pIter);
if( p->rc || pIter->pLeaf==0 ) return;
pIter->iRowid = 0;
@ -4601,6 +4601,7 @@ static void fts5SecureDeleteOverflow(
int iPgno,
int *pbLastInDoclist
){
const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
int pgno;
Fts5Data *pLeaf = 0;
assert( iPgno!=1 );
@ -4627,9 +4628,12 @@ static void fts5SecureDeleteOverflow(
/* The page contains no terms or rowids. Replace it with an empty
** page and move on to the right-hand peer. */
const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
assert_nc( bDetailNone==0 || pLeaf->nn==4 );
if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
fts5DataRelease(pLeaf);
pLeaf = 0;
}else if( bDetailNone ){
break;
}else{
int nShift = iNext - 4;
int nPg;
@ -4678,34 +4682,16 @@ static void fts5SecureDeleteOverflow(
fts5DataRelease(pLeaf);
}
/*
** This is called as part of flushing a delete to disk in 'secure-delete'
** mode. It edits the segments within the database described by argument
** pStruct to remove the entries for term zTerm, rowid iRowid.
** Completely remove the entry that pSeg currently points to from
** the database.
*/
static void fts5FlushSecureDelete(
static void fts5DoSecureDelete(
Fts5Index *p,
Fts5Structure *pStruct,
const char *zTerm,
i64 iRowid
Fts5SegIter *pSeg
){
const int f = FTS5INDEX_QUERY_SKIPHASH;
int nTerm = strlen(zTerm);
Fts5Iter *pIter = 0; /* Used to find term instance */
fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
if( fts5MultiIterEof(p, pIter)==0 ){
i64 iThis = fts5MultiIterRowid(pIter);
if( iThis<iRowid ){
fts5MultiIterNextFrom(p, pIter, iRowid);
}
if( p->rc==SQLITE_OK
&& fts5MultiIterEof(p, pIter)==0
&& iRowid==fts5MultiIterRowid(pIter)
){
Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
int iSegid = pSeg->pSeg->iSegid;
u8 *aPg = pSeg->pLeaf->p;
int nPg = pSeg->pLeaf->nn;
@ -4715,37 +4701,41 @@ static void fts5FlushSecureDelete(
u64 iNextDelta = 0;
int iNextOff = 0;
int iOff = 0;
int nMove = 0;
int nIdx = 0;
u8 *aIdx = 0;
nIdx = nPg-iPgIdx;
aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
if( aIdx ){
int bLastInDoclist = 0;
int iIdx = 0;
int iStart = 0;
int iKeyOff = 0;
int iPrevKeyOff = 0;
int nShift = 0;
int iDelKeyOff = 0; /* Offset of deleted key, if any */
nIdx = nPg-iPgIdx;
aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
if( p->rc ) return;
memcpy(aIdx, &aPg[iPgIdx], nIdx);
/* At this point segment iterator pSeg points to the entry
** this function should remove from the b-tree segment.
**
** More specifically, pSeg->iLeafOffset is the offset of the
** first byte in the position-list for the entry to remove.
** Immediately before this comes two varints that will also
** In detail=full or detail=column mode, pSeg->iLeafOffset is the
** offset of the first byte in the position-list for the entry to
** remove. Immediately before this comes two varints that will also
** need to be removed:
**
** + the rowid or delta rowid value for the entry, and
** + the size of the position list in bytes.
**
** Or, in detail=none mode, there is a single varint prior to
** pSeg->iLeafOffset - the rowid or delta rowid value.
**
** This block sets the following variables:
**
** iStart:
** iDelta:
*/
{
int iSOP;
int nPos = 0;
if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
iStart = pSeg->iTermLeafOffset;
}else{
@ -4753,18 +4743,34 @@ static void fts5FlushSecureDelete(
}
iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
assert_nc( iSOP<=pSeg->iLeafOffset );
if( bDetailNone ){
while( iSOP<pSeg->iLeafOffset ){
if( aPg[iSOP]==0x00 ) iSOP++;
if( aPg[iSOP]==0x00 ) iSOP++;
iStart = iSOP;
iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
}
iNextOff = iSOP;
if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
}else{
int nPos = 0;
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
while( iSOP<pSeg->iLeafOffset ){
iStart = iSOP + (nPos/2);
iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
}
assert_nc( iSOP==pSeg->iLeafOffset );
iNextOff = pSeg->iLeafOffset + pSeg->nPos;
}
}
iOff = iStart;
iNextOff = pSeg->iLeafOffset + pSeg->nPos;
if( iNextOff>=iPgIdx ){
int pgno = pSeg->iLeafPgno+1;
fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
@ -4831,7 +4837,7 @@ static void fts5FlushSecureDelete(
}
iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
if( nPrefix2>nPrefix ){
memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix);
memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
iOff += (nPrefix2-nPrefix);
}
memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
@ -4883,7 +4889,9 @@ static void fts5FlushSecureDelete(
}
if( p->rc==SQLITE_OK ){
nMove = nPg - iNextOff;
const int nMove = nPg - iNextOff;
int nShift = 0;
memmove(&aPg[iOff], &aPg[iNextOff], nMove);
iPgIdx -= (iNextOff - iOff);
nPg = iPgIdx;
@ -4913,6 +4921,35 @@ static void fts5FlushSecureDelete(
}
sqlite3_free(aIdx);
}
/*
** This is called as part of flushing a delete to disk in 'secure-delete'
** mode. It edits the segments within the database described by argument
** pStruct to remove the entries for term zTerm, rowid iRowid.
*/
static void fts5FlushSecureDelete(
Fts5Index *p,
Fts5Structure *pStruct,
const char *zTerm,
i64 iRowid
){
const int f = FTS5INDEX_QUERY_SKIPHASH;
int nTerm = strlen(zTerm);
Fts5Iter *pIter = 0; /* Used to find term instance */
fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
if( fts5MultiIterEof(p, pIter)==0 ){
i64 iThis = fts5MultiIterRowid(pIter);
if( iThis<iRowid ){
fts5MultiIterNextFrom(p, pIter, iRowid);
}
if( p->rc==SQLITE_OK
&& fts5MultiIterEof(p, pIter)==0
&& iRowid==fts5MultiIterRowid(pIter)
){
Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
fts5DoSecureDelete(p, pStruct, pSeg);
}
}
@ -4998,13 +5035,26 @@ static void fts5FlushOneHash(Fts5Index *p){
/* If in secure delete mode, and if this entry in the poslist is
** in fact a delete, then edit the existing segments directly
** using fts5FlushSecureDelete(). */
if( bSecureDelete && (pDoclist[iOff] & 0x01) ){
if( bSecureDelete ){
if( eDetail==FTS5_DETAIL_NONE ){
if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
iOff++;
if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
iOff++;
nDoclist = 0;
}else{
continue;
}
}
}else if( (pDoclist[iOff] & 0x01) ){
fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
iOff++;
continue;
}
}
}
if( p->rc==SQLITE_OK && bTermWritten==0 ){
fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);

14
ext/fts5/test/fts5secure4.test
View File

@ -90,18 +90,20 @@ do_catchsql_test 1.12 {
foreach {tn d} {1 full 2 none 3 column} {
reset_db
do_execsql_test 2.$tn.1 "
CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2');
CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20');
INSERT INTO x1(x1, rank) VALUES('pgsz', 64);
INSERT INTO x1(x1, rank) VALUES('secure-delete', 1);
"
do_execsql_test 2.$tn.2 {
BEGIN;
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
COMMIT;
INSERT INTO x1(x1) VALUES('integrity-check');
}
do_execsql_test 2.$tn.3 {

129
ext/fts5/test/fts5secure5.test Normal file
View File

@ -0,0 +1,129 @@
# 2023 April 14
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
return_if_no_fts5
set ::testprefix fts5secure5
return_if_no_fts5
proc dump {} {
execsql_pp {
SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data
}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
}
do_execsql_test 1.1 {
BEGIN;
INSERT INTO ft1(rowid, a) VALUES(1, 'abcd');
INSERT INTO ft1(rowid, a) VALUES(2, 'abcd');
INSERT INTO ft1(rowid, a) VALUES(3, 'abcd');
COMMIT;
}
do_execsql_test 1.2 {
DELETE FROM ft1 WHERE rowid=1;
}
do_execsql_test 1.3 {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
do_execsql_test 1.4 {
DELETE FROM ft1 WHERE rowid=3;
}
do_execsql_test 1.5 {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
do_execsql_test 1.6 {
DELETE FROM ft1 WHERE rowid=3;
}
do_execsql_test 1.7 {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
}
do_execsql_test 2.1 {
BEGIN;
INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one');
INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two');
INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two');
INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two');
INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three');
COMMIT;
}
do_execsql_test 2.2a {
DELETE FROM ft1 WHERE rowid=3;
}
do_execsql_test 2.2b {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
do_execsql_test 2.3a {
DELETE FROM ft1 WHERE rowid=2;
}
do_execsql_test 2.3b {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
do_execsql_test 2.4a {
DELETE FROM ft1 WHERE rowid=4;
}
do_execsql_test 2.4b {
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1);
INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
}
do_execsql_test 3.1 {
BEGIN;
INSERT INTO ft1(a) VALUES('c');
COMMIT;
}
do_execsql_test 3.2 {
DELETE FROM ft1 WHERE rowid IN (1);
INSERT INTO ft1(ft1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
WITH s(i) AS (
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
)
INSERT INTO ft1 SELECT 'abcdefg' FROM s;
}
do_test 4.1 {
for {set i 500} {$i > 0} {incr i -1} {
execsql { DELETE FROM ft1 WHERE rowid=$i }
execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') }
}
} {}
finish_test

15
manifest
View File

@ -1,5 +1,5 @@
C Fix\sproblems\shandling\scorrupt\srecords\sin\snew\sfts5\scode.
D 2023-04-17T11:41:42.438
C Fix\ssome\sproblems\swith\susing\sfts5\soptions\s'secure-delete'\sand\sdetail=none\stogether.
D 2023-04-17T18:32:22.329
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd
F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989
F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
F ext/fts5/fts5_index.c 13c4ea17f9e6b155d3c6d78b54ae4e5ed22e1692041958f3ec71a5712172d915
F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce
F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@ -190,7 +190,8 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33
F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64
F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc
F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
F ext/fts5/test/fts5secure4.test 58638eab7e80571649d08673e7924d85ec293566377a9e7f49160857f001d529
F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da
F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8
R e73db0b2fec49cf21caca2b40145b9f4
P 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
R 94f7d6e08b83dc7ea2e9a235119af696
U dan
Z 8d23b3253d2a52bc5cc6c258ff86d3c2
Z 7915ab356b23565aa915e2ae0ad516a8
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed