From 6bb9889ef023ec853e1943c45940d8929f98fa0d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 May 2014 20:04:48 +0000 Subject: [PATCH 01/11] Experimental code to prevent FTS indexes from growing indefinitely as the table is updated. FossilOrigin-Name: b3b505a4dd0c679437a4272109f1188175088cd1 --- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 161 +++++++++++++++++++++++++++++++---- manifest | 20 +++-- manifest.uuid | 2 +- test/fts4growth.test | 191 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 352 insertions(+), 24 deletions(-) create mode 100644 test/fts4growth.test diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index a5bb2f0041..171a75d124 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -216,7 +216,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 90d1609226..b54d61390a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -193,6 +193,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -268,6 +269,10 @@ struct SegmentNode { #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -369,7 +374,17 @@ static int fts3SqlStmt( /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?", +/* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -1910,6 +1925,7 @@ static int fts3WriteSegdir( sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ @@ -1920,7 +1936,13 @@ static int fts3WriteSegdir( sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -2246,6 +2268,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -2317,13 +2342,13 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; @@ -2942,6 +2967,106 @@ void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = iVal; + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + int iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 1; + int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + + sqlite3_bind_int(pRange, 1, iAbsLevel+1); + sqlite3_bind_int(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize>nByte ){ + bOk = 0; + break; + } + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -3026,6 +3151,9 @@ static int fts3SegmentMerge( if( rc!=SQLITE_OK ) goto finished; } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( rc==SQLITE_OK ){ + rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + } finished: fts3SegWriterFree(pWriter); @@ -3035,7 +3163,7 @@ static int fts3SegmentMerge( /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; @@ -3426,6 +3554,7 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -3764,8 +3893,8 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; @@ -3864,6 +3993,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + pWriter->nLeafData, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -3965,7 +4095,7 @@ static int fts3IncrmergeLoad( if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4566,11 +4696,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ @@ -4687,6 +4817,9 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); diff --git a/manifest b/manifest index 00a1115ee6..1ff6b237e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\susing\sVisual\sStudio\s2013,\sadd\sthe\sappropriate\sMaxPlatformVersion\sattribute\sto\sthe\sVSIX\sSDK\smanifest. -D 2014-05-10T17:28:45.158 +C Experimental\scode\sto\sprevent\sFTS\sindexes\sfrom\sgrowing\sindefinitely\sas\sthe\stable\sis\supdated. +D 2014-05-12T20:04:48.571 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -80,7 +80,7 @@ F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 +F ext/fts3/fts3Int.h b250b154764b86be4ee540f8c233a59858e88f1d F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934 +F ext/fts3/fts3_write.c 5fd2aa9d1812387c6254304e20d9ac2b29e16700 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,6 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 +F test/fts4growth.test f7eac9fadfe67765c4a0d6202c85f7272766fb9e F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1170,7 +1171,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c3dce2e7390eec3a337be1b99f80ad5f721cc647 -R 7cb8e1b3df65a486b753c19d33da3975 -U mistachkin -Z f2874f77f50d8a6763121ff9dede6833 +P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 +R de45a14af3a90f6b390273a9a2d8cfa9 +T *branch * fts4-experimental +T *sym-fts4-experimental * +T -sym-trunk * +U dan +Z a03d85f9bfa278d813164b1e97a88ff7 diff --git a/manifest.uuid b/manifest.uuid index ff047bf0ce..a9facd1d8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 \ No newline at end of file +b3b505a4dd0c679437a4272109f1188175088cd1 \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test new file mode 100644 index 0000000000..ab2ad62142 --- /dev/null +++ b/test/fts4growth.test @@ -0,0 +1,191 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; } + +do_test 1.2 { + foreach L { + {"See here, young man," said Mulga Bill, "from Walgett to the sea,} + {From Conroy's Gap to Castlereagh, there's none can ride like me.} + {I'm good all round at everything as everybody knows,} + {Although I'm not the one to talk -- I hate a man that blows.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { SELECT end_block, length(root) FROM x1_segdir } +} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115} + +do_execsql_test 1.3 { + INSERT INTO x1(x1) VALUES('optimize'); + SELECT level, end_block, length(root) FROM x1_segdir; +} {0 {0 394} 394} + +do_test 1.4 { + foreach L { + {But riding is my special gift, my chiefest, sole delight;} + {Just ask a wild duck can it swim, a wildcat can it fight.} + {There's nothing clothed in hair or hide, or built of flesh or steel,} + {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,} + {But what I'll sit, while hide will hold and girths and straps are tight:} + {I'll ride this here two-wheeled concern right straight away at sight."} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + INSERT INTO x1(x1) VALUES('merge=4,4'); + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2} + +do_execsql_test 1.5 { + SELECT length(block) FROM x1_segments; +} {658 {}} + +do_test 1.6 { + foreach L { + {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,} + {That perched above Dead Man's Creek, beside the mountain road.} + {He turned the cycle down the hill and mounted for the fray,} + {But 'ere he'd gone a dozen yards it bolted clean away.} + {It left the track, and through the trees, just like a silver steak,} + {It whistled down the awful slope towards the Dead Man's Creek.} + {It shaved a stump by half an inch, it dodged a big white-box:} + {The very wallaroos in fright went scrambling up the rocks,} + {The wombats hiding in their caves dug deeper underground,} + {As Mulga Bill, as white as chalk, sat tight to every bound.} + {It struck a stone and gave a spring that cleared a fallen tree,} + {It raced beside a precipice as close as close could be;} + {And then as Mulga Bill let out one last despairing shriek} + {It made a leap of twenty feet into the Dead Man's Creek.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117} + +do_execsql_test 1.7 { + SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130); +} {1377} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis +do_test 2.2 { + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id + } + } + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id + } + if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break + } +} {} + +do_execsql_test 2.3 { + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 0} + +do_execsql_test 2.4 { + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 1} + +do_execsql_test 2.5 { + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 3430} {3828 10191} {3828 14109}} + +do_execsql_test 2.6 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {14109} + +do_execsql_test 2.7 { + INSERT INTO x2(x2) VALUES('merge=1000,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 86120}} + +do_execsql_test 2.8 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {86120} + +#-------------------------------------------------------------------------- + +do_execsql_test 3.1 { + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis + +proc t1_to_x2 {} { + foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { + execsql { + DELETE FROM x2 WHERE docid=$id; + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id; + } + } +} + +#do_test 3.2 { + #t1_to_x2 + #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} +#} {0 13 1 15 2 5} + +#proc second {x} { lindex $x 1 } +#db func second second +#for {set i 0} {$i <1000} {incr i} { +# t1_to_x2 +# db eval { +# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; +# } { +# puts "$i.$level: $c" +# } +#} + + +finish_test + + From c0caea21dee91b1f80745bf9b81a07b237ec314e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 May 2014 20:11:37 +0000 Subject: [PATCH 02/11] Fix a problem preventing delete markers from ever being removed from the FTS index. FossilOrigin-Name: 7f47ae5c5ddb1227484ddae7c6960183932a052a --- ext/fts3/fts3_write.c | 54 ++++++++++------- manifest | 19 +++--- manifest.uuid | 2 +- test/fts3d.test | 19 +++--- test/fts4growth.test | 132 ++++++++++++++++++++++++++++++++++++++---- 5 files changed, 174 insertions(+), 52 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index b54d61390a..e2093376e6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -382,8 +382,9 @@ static int fts3SqlStmt( "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ -/* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?", -/* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1" +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" }; int rc = SQLITE_OK; @@ -3091,6 +3092,7 @@ static int fts3SegmentMerge( Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -3102,6 +3104,11 @@ static int fts3SegmentMerge( rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically @@ -3111,21 +3118,21 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + if( pWriter ){ + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( rc==SQLITE_OK ){ + rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + } } finished: @@ -4725,6 +4734,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. @@ -4778,6 +4788,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } @@ -4785,16 +4801,12 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ diff --git a/manifest b/manifest index 1ff6b237e6..357a99fd07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode\sto\sprevent\sFTS\sindexes\sfrom\sgrowing\sindefinitely\sas\sthe\stable\sis\supdated. -D 2014-05-12T20:04:48.571 +C Fix\sa\sproblem\spreventing\sdelete\smarkers\sfrom\sever\sbeing\sremoved\sfrom\sthe\sFTS\sindex. +D 2014-05-13T20:11:37.423 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 5fd2aa9d1812387c6254304e20d9ac2b29e16700 +F ext/fts3/fts3_write.c 283b24477729129a210d91b48f7c53181583a848 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -539,7 +539,7 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 -F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f +F test/fts3d.test 95c17d1b67b33a5eac0bf5a0d11116a0c0ac7a3a F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd @@ -570,7 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test f7eac9fadfe67765c4a0d6202c85f7272766fb9e +F test/fts4growth.test 3b1f8c98b603b38dc9fe4a266f4f5ddb0c73f092 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1171,10 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7 -R de45a14af3a90f6b390273a9a2d8cfa9 -T *branch * fts4-experimental -T *sym-fts4-experimental * -T -sym-trunk * +P b3b505a4dd0c679437a4272109f1188175088cd1 +R d1a058fc899c4ecbd09fd7fceab935e7 U dan -Z a03d85f9bfa278d813164b1e97a88ff7 +Z f678f48fb654227badac5ce98e23d62c diff --git a/manifest.uuid b/manifest.uuid index a9facd1d8e..a6856e6014 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3b505a4dd0c679437a4272109f1188175088cd1 \ No newline at end of file +7f47ae5c5ddb1227484ddae7c6960183932a052a \ No newline at end of file diff --git a/test/fts3d.test b/test/fts3d.test index 2914818d4e..5c04ead0a0 100644 --- a/test/fts3d.test +++ b/test/fts3d.test @@ -213,16 +213,17 @@ do_test fts3d-4.matches { {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] -check_terms_all fts3d-4.1 {a four is one test that this three two was} +puts [db eval {SELECT c FROM t1 } ] +check_terms_all fts3d-4.1 {a four is test that this was} check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts3d-4.1.2 four {} check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]} -check_doclist_all fts3d-4.1.4 one {} +#check_doclist_all fts3d-4.1.4 one {} check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts3d-4.1.6 that {[2 0[0]]} check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]} -check_doclist_all fts3d-4.1.8 three {} -check_doclist_all fts3d-4.1.9 two {} +#check_doclist_all fts3d-4.1.8 three {} +#check_doclist_all fts3d-4.1.9 two {} check_doclist_all fts3d-4.1.10 was {[2 0[1]]} check_terms fts3d-4.2 0 0 {a four test that was} @@ -239,16 +240,16 @@ check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]} check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]} check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]} -check_terms fts3d-4.4 1 0 {a four is one test that this three two was} +check_terms fts3d-4.4 1 0 {a four is test that this was} check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]} -check_doclist fts3d-4.4.2 1 0 four {[1] [2 0[4]] [3 0[4]]} +check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]} check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]} -check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} +#check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]} check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} -check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} -check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} +#check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} +#check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level diff --git a/test/fts4growth.test b/test/fts4growth.test index ab2ad62142..e5b419a389 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -151,7 +151,118 @@ do_execsql_test 2.8 { } {86120} #-------------------------------------------------------------------------- +# Test that delete markers are removed from FTS segments when possible. +# It is only possible to remove delete markers when the output of the +# merge operation will become the oldest segment in the index. +# +# 3.1 - when the oldest segment is created by an 'optimize'. +# 3.2 - when the oldest segment is created by an incremental merge. +# 3.3 - by a crisis merge. +# +proc insert_doc {args} { + foreach iDoc $args { + set L [lindex { + {In your eagerness to engage the Trojans,} + {don’t any of you charge ahead of others,} + {trusting in your strength and horsemanship.} + {And don’t lag behind. That will hurt our charge.} + {Any man whose chariot confronts an enemy’s} + {should thrust with his spear at him from there.} + {That’s the most effective tactic, the way} + {men wiped out city strongholds long ago —} + {their chests full of that style and spirit.} + } [expr $iDoc%9]] + execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) } + } +} + +proc delete_doc {args} { + foreach iDoc $args { + execsql { DELETE FROM x3 WHERE docid = $iDoc } + } +} + +proc second {x} { lindex $x 1 } +db func second second + +do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 } + +do_test 3.1.1 { + db transaction { insert_doc 1 2 3 4 5 6 } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} +do_test 3.1.2 { + delete_doc 1 2 3 4 5 6 + execsql { SELECT count(*) FROM x3_segdir } +} {0} +do_test 3.1.3 { + db transaction { + insert_doc 1 2 3 4 5 6 7 8 9 + delete_doc 9 8 7 + } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 591 0 1 65 0 2 72 0 3 76} +do_test 3.1.4 { + execsql { INSERT INTO x3(x3) VALUES('optimize') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} + +do_test 3.2.1 { + execsql { DELETE FROM x3 } + insert_doc 8 7 6 5 4 3 2 1 + delete_doc 7 8 + execsql { SELECT count(*) FROM x3_segdir } +} {10} +do_test 3.2.2 { + execsql { INSERT INTO x3(x3) VALUES('merge=500,10') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {1 0 412} + +# This assumes the crisis merge happens when there are already 16 +# segments and one more is added. +# +do_test 3.3.1 { + execsql { DELETE FROM x3 } + insert_doc 1 2 3 4 5 6 7 8 9 10 11 + delete_doc 11 10 9 8 7 + execsql { SELECT count(*) FROM x3_segdir } +} {16} + +do_test 3.3.2 { + insert_doc 12 + execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 } +} {1 0 412} + +#-------------------------------------------------------------------------- +do_execsql_test 4.1 { + DROP TABLE IF EXISTS x4; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x4 USING fts4(words); +} +do_test 4.2 { + fts_kjv_genesis + execsql { INSERT INTO x4 SELECT words FROM t1 } + execsql { INSERT INTO x4 SELECT words FROM t1 } +} {} + +do_execsql_test 4.3 { + SELECT level, idx, second(end_block) FROM x4_segdir +} {0 0 117483 0 1 118006} + +do_execsql_test 4.4 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +breakpoint +do_execsql_test 4.5 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +if 0 { do_execsql_test 3.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; @@ -174,16 +285,17 @@ proc t1_to_x2 {} { #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} #} {0 13 1 15 2 5} -#proc second {x} { lindex $x 1 } -#db func second second -#for {set i 0} {$i <1000} {incr i} { -# t1_to_x2 -# db eval { -# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; -# } { -# puts "$i.$level: $c" -# } -#} +proc second {x} { lindex $x 1 } +db func second second +for {set i 0} {$i <1000} {incr i} { + t1_to_x2 + db eval { + SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; + } { + puts "$i.$level: $c" + } +} +} finish_test From 9e83e465eba9417372f45f5d87b2cd3777690a7a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 15:58:47 +0000 Subject: [PATCH 03/11] Fix various problems to do with segment promotion. Add test file fts4growth2.test, containing tests to check that the FTS index does not grow indefinitely as the table is updated. FossilOrigin-Name: 21491a9bc686e63bec32f1a67103622f1160a2f9 --- ext/fts3/fts3.c | 9 ++-- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 60 ++++++++++++++++++--- manifest | 21 ++++---- manifest.uuid | 2 +- test/fts4growth.test | 117 +++++++++++++++++++++++++++++++++++------ test/fts4growth2.test | 89 +++++++++++++++++++++++++++++++ test/permutations.test | 2 + 8 files changed, 262 insertions(+), 40 deletions(-) create mode 100644 test/fts4growth2.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f7b2232902..c421aa9229 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1333,7 +1333,7 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -3302,7 +3302,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ @@ -3310,7 +3313,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 171a75d124..1383102f4c 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -210,7 +210,7 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index e2093376e6..1e82187d44 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2971,16 +2971,23 @@ void sqlite3Fts3SegReaderFinish( /* ** Decode the "end_block" field, selected by column iCol of the SELECT ** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. */ static void fts3ReadEndBlockField( sqlite3_stmt *pStmt, int iCol, - i64 *piEndBlock, + i64 *piEndBlock, i64 *pnByte ){ const unsigned char *zText = sqlite3_column_text(pStmt, iCol); if( zText ){ int i; + int iMul = 1; i64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); @@ -2988,10 +2995,14 @@ static void fts3ReadEndBlockField( *piEndBlock = iVal; while( zText[i]==' ' ) i++; iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *pnByte = iVal; + *pnByte = (iVal * (i64)iMul); } } @@ -3011,18 +3022,30 @@ static int fts3PromoteSegments( rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); if( rc==SQLITE_OK ){ - int bOk = 1; + int bOk = 0; int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ sqlite3_bind_int(pRange, 1, iAbsLevel+1); sqlite3_bind_int(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); - if( nSize>nByte ){ + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ bOk = 0; break; } + bOk = 1; } rc = sqlite3_reset(pRange); @@ -3039,6 +3062,16 @@ static int fts3PromoteSegments( } if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ sqlite3_bind_int(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); @@ -3056,6 +3089,7 @@ static int fts3PromoteSegments( rc = sqlite3_reset(pRange); } + /* Move level -1 to level iAbsLevel */ if( rc==SQLITE_OK ){ sqlite3_bind_int(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); @@ -3188,14 +3222,15 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + p->nAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; rc = sqlite3_reset(pStmt); } } @@ -4105,6 +4140,9 @@ static int fts3IncrmergeLoad( iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4828,6 +4866,9 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); if( nSeg==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); @@ -4918,7 +4959,10 @@ static int fts3DoAutoincrmerge( ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); @@ -4927,7 +4971,7 @@ static int fts3DoAutoincrmerge( rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; diff --git a/manifest b/manifest index 357a99fd07..276a411569 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\spreventing\sdelete\smarkers\sfrom\sever\sbeing\sremoved\sfrom\sthe\sFTS\sindex. -D 2014-05-13T20:11:37.423 +C Fix\svarious\sproblems\sto\sdo\swith\ssegment\spromotion.\sAdd\stest\sfile\sfts4growth2.test,\scontaining\stests\sto\scheck\sthat\sthe\sFTS\sindex\sdoes\snot\sgrow\sindefinitely\sas\sthe\stable\sis\supdated. +D 2014-05-14T15:58:47.565 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db +F ext/fts3/fts3.c e83f894cf1adaf8decd6b1de76bfdcdb79b25507 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h b250b154764b86be4ee540f8c233a59858e88f1d +F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 2ac35bda474f00c14c19608e49a02c8c7ceb9970 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 283b24477729129a210d91b48f7c53181583a848 +F ext/fts3/fts3_write.c 5e13a15dad44bf516181e9d6251c7d6a27861dee F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,7 +570,8 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test 3b1f8c98b603b38dc9fe4a266f4f5ddb0c73f092 +F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 +F test/fts4growth2.test 19fda88600266629b63a4cece512c2dc6f7c9fc5 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -745,7 +746,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 +F test/permutations.test 975f65293a46cf8deb896ecdaf1ba563ccaca24a F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1171,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3b505a4dd0c679437a4272109f1188175088cd1 -R d1a058fc899c4ecbd09fd7fceab935e7 +P 7f47ae5c5ddb1227484ddae7c6960183932a052a +R a0acf9231899e84a47fa0a8ff85a8e05 U dan -Z f678f48fb654227badac5ce98e23d62c +Z 4a0ad803ddf94719a771980af32709c7 diff --git a/manifest.uuid b/manifest.uuid index a6856e6014..4b5569d5e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f47ae5c5ddb1227484ddae7c6960183932a052a \ No newline at end of file +21491a9bc686e63bec32f1a67103622f1160a2f9 \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test index e5b419a389..466416e561 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -131,7 +131,7 @@ do_execsql_test 2.5 { SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; -} {{3828 3430} {3828 10191} {3828 14109}} +} {{3828 -3430} {3828 -10191} {3828 -14109}} do_execsql_test 2.6 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE @@ -235,6 +235,9 @@ do_test 3.3.2 { } {1 0 412} #-------------------------------------------------------------------------- +# Check a theory on a bug in fts4 - that segments with idx==0 were not +# being incrementally merged correctly. Theory turned out to be false. +# do_execsql_test 4.1 { DROP TABLE IF EXISTS x4; DROP TABLE IF EXISTS t1; @@ -256,14 +259,23 @@ do_execsql_test 4.4 { SELECT count(*) FROM x4_segdir; } {3} -breakpoint do_execsql_test 4.5 { INSERT INTO x4(x4) VALUES('merge=10,2'); SELECT count(*) FROM x4_segdir; } {3} -if 0 { -do_execsql_test 3.1 { +do_execsql_test 4.6 { + INSERT INTO x4(x4) VALUES('merge=1000,2'); + SELECT count(*) FROM x4_segdir; +} {1} + + + +#-------------------------------------------------------------------------- +# Check that segments are not promoted if the "end_block" field does not +# contain a size. +# +do_execsql_test 5.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; CREATE TABLE t1(docid, words); @@ -271,6 +283,41 @@ do_execsql_test 3.1 { } fts_kjv_genesis +proc first {L} {lindex $L 0} +db func first first + +do_test 5.2 { + foreach r [db eval { SELECT rowid FROM t1 }] { + execsql { + INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r + } + } + foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] { + execsql { DELETE FROM x2 WHERE docid = $d } + } + + execsql { + INSERT INTO x2(x2) VALUES('optimize'); + SELECT level, idx, end_block FROM x2_segdir + } +} {2 0 {752 1926}} + +do_execsql_test 5.3 { + UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER ); + SELECT end_block, typeof(end_block) FROM x2_segdir; +} {752 integer} + +do_execsql_test 5.4 { + INSERT INTO x2 SELECT words FROM t1 LIMIT 50; + SELECT level, idx, end_block FROM x2_segdir +} {2 0 752 0 0 {758 5174}} + +do_execsql_test 5.5 { + UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2; + INSERT INTO x2 SELECT words FROM t1 LIMIT 40; + SELECT level, idx, end_block FROM x2_segdir +} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}} + proc t1_to_x2 {} { foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { execsql { @@ -280,24 +327,60 @@ proc t1_to_x2 {} { } } +#-------------------------------------------------------------------------- +# Check that segments created by auto-merge are not promoted until they +# are completed. +# + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE x5 USING fts4; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75; + SELECT count(*) FROM x5_segdir +} {4} + +do_execsql_test 6.2 { + INSERT INTO x5(x5) VALUES('merge=2,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}} + +do_execsql_test 6.3 { + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} + 0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297} +} + +do_execsql_test 6.4 { + INSERT INTO x5(x5) VALUES('merge=200,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {1329 8297} 1 0 {1320 28009}} + +do_execsql_test 6.5 { + INSERT INTO x5 SELECT words FROM t1; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} +} + #do_test 3.2 { #t1_to_x2 #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} #} {0 13 1 15 2 5} -proc second {x} { lindex $x 1 } -db func second second -for {set i 0} {$i <1000} {incr i} { - t1_to_x2 - db eval { - SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; - } { - puts "$i.$level: $c" - } -} -} - +#proc second {x} { lindex $x 1 } +#db func second second +#for {set i 0} {$i <1000} {incr i} { +# t1_to_x2 +# db eval { +# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; +# } { +# puts "$i.$level: $c" +# } +#} finish_test - diff --git a/test/fts4growth2.test b/test/fts4growth2.test new file mode 100644 index 0000000000..e56309ab36 --- /dev/null +++ b/test/fts4growth2.test @@ -0,0 +1,89 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } +fts_kjv_genesis + +proc tt {val} { + execsql { + BEGIN; + DELETE FROM x1 + WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + INSERT INTO x1(docid, content) + SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; + COMMIT; + } +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE x1 USING fts4; + INSERT INTO x1(x1) VALUES('automerge=2'); +} + +do_test 1.2 { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=4; + } +} {4 1} + +do_test 1.3 { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=4; + } +} {4 1} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + DELETE FROM t1 WHERE rowid>16; + DROP TABLE IF EXISTS x1; + CREATE VIRTUAL TABLE x1 USING fts4; +} + +db func second second +proc second {L} {lindex $L 1} + +for {set tn 0} {$tn < 40} {incr tn} { + do_test 2.2.$tn { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { SELECT max(level) FROM x1_segdir } + } {1} +} + + +finish_test + diff --git a/test/permutations.test b/test/permutations.test index 7f1485f831..b7e1eaf8d2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,6 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + fts3growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] @@ -196,6 +197,7 @@ test_suite "fts3" -prefix "" -description { fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test + fts4growth.test fts4growth2.test } test_suite "nofaultsim" -prefix "" -description { From a0b7ca90b71fb47e370ecb4b2ec2f27b01dc7617 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 18:45:27 +0000 Subject: [PATCH 04/11] Fix a problem to do with loading the value of the FTS automerge setting from the database. FossilOrigin-Name: 7268119f74602929b372b88eca7b3c3b4964f7d5 --- ext/fts3/fts3_write.c | 6 ++++-- manifest | 18 ++++++++--------- manifest.uuid | 2 +- test/fts4growth2.test | 25 ++++++++++++------------ test/fts4merge4.test | 44 ++++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 2 +- 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1e82187d44..9bc2091d20 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3229,8 +3229,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->nAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); - if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + } rc = sqlite3_reset(pStmt); } } diff --git a/manifest b/manifest index 276a411569..8a696af535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sproblems\sto\sdo\swith\ssegment\spromotion.\sAdd\stest\sfile\sfts4growth2.test,\scontaining\stests\sto\scheck\sthat\sthe\sFTS\sindex\sdoes\snot\sgrow\sindefinitely\sas\sthe\stable\sis\supdated. -D 2014-05-14T15:58:47.565 +C Fix\sa\sproblem\sto\sdo\swith\sloading\sthe\svalue\sof\sthe\sFTS\sautomerge\ssetting\sfrom\sthe\sdatabase. +D 2014-05-14T18:45:27.234 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 5e13a15dad44bf516181e9d6251c7d6a27861dee +F ext/fts3/fts3_write.c 6cb17b13b08fa2f36c8079726207736246a7176c F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -571,13 +571,13 @@ F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 -F test/fts4growth2.test 19fda88600266629b63a4cece512c2dc6f7c9fc5 +F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 -F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 +F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d @@ -746,7 +746,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 975f65293a46cf8deb896ecdaf1ba563ccaca24a +F test/permutations.test d8946218493c77806ec1029df473d8ed4ad9aef6 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f47ae5c5ddb1227484ddae7c6960183932a052a -R a0acf9231899e84a47fa0a8ff85a8e05 +P 21491a9bc686e63bec32f1a67103622f1160a2f9 +R 67348fc4be2e8c129b7e6e89a490a40b U dan -Z 4a0ad803ddf94719a771980af32709c7 +Z e904ec03ef68eb7c18f9e542f40d2d02 diff --git a/manifest.uuid b/manifest.uuid index 4b5569d5e0..698af193d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21491a9bc686e63bec32f1a67103622f1160a2f9 \ No newline at end of file +7268119f74602929b372b88eca7b3c3b4964f7d5 \ No newline at end of file diff --git a/test/fts4growth2.test b/test/fts4growth2.test index e56309ab36..023ca8ba1c 100644 --- a/test/fts4growth2.test +++ b/test/fts4growth2.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix fts4growth +set testprefix fts4growth2 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { @@ -30,13 +30,12 @@ fts_kjv_genesis proc tt {val} { execsql { - BEGIN; - DELETE FROM x1 - WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); - INSERT INTO x1(docid, content) - SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; - COMMIT; + DELETE FROM x1 + WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + INSERT INTO x1(docid, content) + SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; } + #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] } do_execsql_test 1.1 { @@ -45,24 +44,24 @@ do_execsql_test 1.1 { } do_test 1.2 { - for {set i 0} {$i < 100} {incr i} { + for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=4; + SELECT count(*) FROM x1_segdir WHERE level=3; } -} {4 1} +} {3 1} do_test 1.3 { - for {set i 0} {$i < 100} {incr i} { + for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=4; + SELECT count(*) FROM x1_segdir WHERE level=2; } -} {4 1} +} {2 1} #------------------------------------------------------------------------- # diff --git a/test/fts4merge4.test b/test/fts4merge4.test index 8e2119de2b..038e460d0e 100644 --- a/test/fts4merge4.test +++ b/test/fts4merge4.test @@ -53,6 +53,50 @@ do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 +#------------------------------------------------------------------------- +# Now test that the automerge=? option appears to work. +# +do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; } + +set doc "" +foreach c1 "a b c d e f g h i j" { + foreach c2 "a b c d e f g h i j" { + foreach c3 "a b c d e f g h i j" { + lappend doc "$c1$c2$c3" + } + } +} +set doc [string repeat $doc 10] + +foreach {tn am expected} { + 1 {automerge=2} {1 1 2 1 4 1 6 1} + 2 {automerge=4} {1 2 2 1 3 1} + 3 {automerge=8} {0 4 1 3 2 1} + 4 {automerge=1} {0 4 1 3 2 1} +} { + foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} { + do_test 2.2.$tn.$tn2 { + execsql { DELETE FROM t2 } + execsql { INSERT INTO t2(t2) VALUES($am) }; + + eval $openclose + + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + COMMIT; + } + } + + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } + } [list {*}$expected] + } +} sqlite3_enable_shared_cache $::enable_shared_cache finish_test diff --git a/test/permutations.test b/test/permutations.test index b7e1eaf8d2..0e5cc2c8d2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,7 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test - fts3growth2.test + fts4growth.test fts4growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] From 5b34f15a82644307478dc01fdb0c0cda36430d0c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 May 2014 19:49:17 +0000 Subject: [PATCH 05/11] Do not search for promotable segments following an FTS 'optimize' operation or creation of a new segment on the oldest existing level. FossilOrigin-Name: 34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab --- ext/fts3/fts3_write.c | 8 ++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/trace2.test | 1 + 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 9bc2091d20..ce16f0c86a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -378,7 +378,7 @@ static int fts3SqlStmt( /* Return segments in order from oldest to newest.*/ /* 37 */ "SELECT level, idx, end_block " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ @@ -3194,7 +3194,9 @@ static int fts3SegmentMerge( if( pWriter ){ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); if( rc==SQLITE_OK ){ - rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); + if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevelnLeafData); + } } } @@ -3232,6 +3234,8 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ if( rc==SQLITE_ROW ){ p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; } rc = sqlite3_reset(pStmt); } diff --git a/manifest b/manifest index 8a696af535..2b1578669e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sto\sdo\swith\sloading\sthe\svalue\sof\sthe\sFTS\sautomerge\ssetting\sfrom\sthe\sdatabase. -D 2014-05-14T18:45:27.234 +C Do\snot\ssearch\sfor\spromotable\ssegments\sfollowing\san\sFTS\s'optimize'\soperation\sor\screation\sof\sa\snew\ssegment\son\sthe\soldest\sexisting\slevel. +D 2014-05-14T19:49:17.392 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 6cb17b13b08fa2f36c8079726207736246a7176c +F ext/fts3/fts3_write.c e58c8df149436dcc85e2face5516b75a01c7a583 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1008,7 +1008,7 @@ F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 8f4ac52f62f3e9f6bce0889105aecdf0275e331b F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test e7a988fdd982cdec62f1f1f34b0360e6476d01a0 +F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 21491a9bc686e63bec32f1a67103622f1160a2f9 -R 67348fc4be2e8c129b7e6e89a490a40b +P 7268119f74602929b372b88eca7b3c3b4964f7d5 +R 38e0e000cd94731b650020d6321d9a1b U dan -Z e904ec03ef68eb7c18f9e542f40d2d02 +Z 73a15f3066e0e6dc1f73e5ba383c60f7 diff --git a/manifest.uuid b/manifest.uuid index 698af193d2..0656ed2305 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7268119f74602929b372b88eca7b3c3b4964f7d5 \ No newline at end of file +34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab \ No newline at end of file diff --git a/test/trace2.test b/test/trace2.test index 8f68d87585..562c70c538 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -136,6 +136,7 @@ ifcapable fts3 { "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC" } do_trace_test 2.3 { From 157c957a65887a07495397dc95ac27ac3686d77e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 May 2014 18:36:39 +0000 Subject: [PATCH 06/11] Fix a problem with the handling of delete markers by automerge on large databases. FossilOrigin-Name: 2b09bd17eb85ea3c682e930d2aabc6441f2265e0 --- ext/fts3/fts3_write.c | 40 +++++++++++++++++++++++++++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ce16f0c86a..2f940eac7a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2433,6 +2433,37 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents @@ -4836,8 +4867,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK ){ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); assert( bUseHint==1 || bUseHint==0 ); - if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } diff --git a/manifest b/manifest index 2b1578669e..8d7389d529 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssearch\sfor\spromotable\ssegments\sfollowing\san\sFTS\s'optimize'\soperation\sor\screation\sof\sa\snew\ssegment\son\sthe\soldest\sexisting\slevel. -D 2014-05-14T19:49:17.392 +C Fix\sa\sproblem\swith\sthe\shandling\sof\sdelete\smarkers\sby\sautomerge\son\slarge\sdatabases. +D 2014-05-15T18:36:39.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c e58c8df149436dcc85e2face5516b75a01c7a583 +F ext/fts3/fts3_write.c b90173c62460a215498b0da3046f5e429b3a66b0 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7268119f74602929b372b88eca7b3c3b4964f7d5 -R 38e0e000cd94731b650020d6321d9a1b +P 34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab +R e26158b3998e23a04d5e7968b1e6dfac U dan -Z 73a15f3066e0e6dc1f73e5ba383c60f7 +Z 27219917e93f1cfe1a1e1c19124576cc diff --git a/manifest.uuid b/manifest.uuid index 0656ed2305..8f898f62ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab \ No newline at end of file +2b09bd17eb85ea3c682e930d2aabc6441f2265e0 \ No newline at end of file From a7b9e7d2d9e06600d0dc7e4df62e9026ff7053d2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 May 2014 10:30:44 +0000 Subject: [PATCH 07/11] Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. FossilOrigin-Name: a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b --- ext/fts3/fts3_write.c | 6 ++-- manifest | 14 ++++---- manifest.uuid | 2 +- test/fts4growth.test | 79 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 2f940eac7a..2cbf084e84 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3636,6 +3636,7 @@ struct IncrmergeWriter { sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -4074,7 +4075,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ - pWriter->nLeafData, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -4180,6 +4181,7 @@ static int fts3IncrmergeLoad( if( pWriter->nLeafData<0 ){ pWriter->nLeafData = pWriter->nLeafData * -1; } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -4914,7 +4916,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ pWriter->nLeafData = pWriter->nLeafData * -1; } fts3IncrmergeRelease(p, pWriter, &rc); - if( nSeg==0 ){ + if( nSeg==0 && pWriter->bNoLeafData==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); } } diff --git a/manifest b/manifest index f27c4b1f83..f12f2aa4a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2014-05-15T19:05:55.306 +C Fix\sa\sbug\scausing\san\sincorrect\ssegment\ssize\svalue\sto\sbe\sstored\sif\sboth\san\sold\sand\snew\sFTS\sversion\sperformed\swork\son\sthe\ssame\sincremental\smerge\soperation. +D 2014-05-16T10:30:44.076 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c b90173c62460a215498b0da3046f5e429b3a66b0 +F ext/fts3/fts3_write.c 6681e4855bd24e74e91e2017f3fdc44460449fc1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -570,7 +570,7 @@ F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test 50befaa001a6dd4046d32e5d7b3cb75df8360ca8 +F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53 F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b09bd17eb85ea3c682e930d2aabc6441f2265e0 0901dccedaf5679691ba803ec76d113de170185d -R bc36878656a1df406ce5faa15b3690cd +P 5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 +R 12132d8d56290c779558e11739c0e6c3 U dan -Z 7b26c3764a28756f379665cd911542c0 +Z a463ca0b0e1281a342011aafc1248dc1 diff --git a/manifest.uuid b/manifest.uuid index 414ca82c34..0c30c7f481 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 \ No newline at end of file +a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b \ No newline at end of file diff --git a/test/fts4growth.test b/test/fts4growth.test index 466416e561..aa5f251f95 100644 --- a/test/fts4growth.test +++ b/test/fts4growth.test @@ -366,21 +366,72 @@ do_execsql_test 6.5 { 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} } -#do_test 3.2 { - #t1_to_x2 - #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level} -#} {0 13 1 15 2 5} +#-------------------------------------------------------------------------- +# Ensure that if part of an incremental merge is performed by an old +# version that does not support storing segment sizes in the end_block +# field, no size is stored in the final segment (as it would be incorrect). +# +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE x6 USING fts4; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} +} + +do_execsql_test 7.2 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 {16014 -51226} +} + +do_execsql_test 7.3 { + UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.4 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.5 { + INSERT INTO x6(x6) VALUES('merge=2500,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {598 118006} 0 1 {718 118006} 1 0 16014 +} + +do_execsql_test 7.6 { + INSERT INTO x6(x6) VALUES('merge=2500,2'); + SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir; +} { + 2 0 23695 24147 {41262 633507} +} + +do_execsql_test 7.7 { + SELECT sum(length(block)) FROM x6_segments + WHERE blockid BETWEEN 23695 AND 24147 +} {633507} + -#proc second {x} { lindex $x 1 } -#db func second second -#for {set i 0} {$i <1000} {incr i} { -# t1_to_x2 -# db eval { -# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level; -# } { -# puts "$i.$level: $c" -# } -#} finish_test From e82384817d3ce4b1657664bb5233fa573fcd5082 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 May 2014 15:48:17 +0000 Subject: [PATCH 08/11] Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. FossilOrigin-Name: 3997d47bb9c8593a2cd30317ea65163b99db1832 --- manifest | 12 +++++----- manifest.uuid | 2 +- test/backcompat.test | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f12f2aa4a8..543d66fc2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\scausing\san\sincorrect\ssegment\ssize\svalue\sto\sbe\sstored\sif\sboth\san\sold\sand\snew\sFTS\sversion\sperformed\swork\son\sthe\ssame\sincremental\smerge\soperation. -D 2014-05-16T10:30:44.076 +C Add\sextra\stest\sto\sbackcompat.test\sto\sensure\sthat\sold\sand\snew\sversions\sof\sFTS\smay\swork\stogether\son\sthe\ssame\sincremental\smerge\soperation. +D 2014-05-16T15:48:17.688 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -336,7 +336,7 @@ F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 -F test/backcompat.test 5f8ad58b3eaebc78cd2c66c65476a42e6f32b2ad +F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5809986ffcc5d9cbab64760f8d7fbfbb6b626ee7 -R 12132d8d56290c779558e11739c0e6c3 +P a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b +R 03d2cea8f8e26027c7bf9b05231d6fb8 U dan -Z a463ca0b0e1281a342011aafc1248dc1 +Z ba763e498b0d066dca73ae5326487971 diff --git a/manifest.uuid b/manifest.uuid index 0c30c7f481..1f33001479 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b \ No newline at end of file +3997d47bb9c8593a2cd30317ea65163b99db1832 \ No newline at end of file diff --git a/test/backcompat.test b/test/backcompat.test index fdc2410b7a..ea7e6a9eed 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -58,12 +58,24 @@ proc do_backcompat_test {rv bin1 bin2 script} { code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } + foreach c {code1 code2} { + $c { + set v [split [db version] .] + if {[llength $v]==3} {lappend v 0} + set ::sqlite_libversion [format \ + "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3] + ] + } + } + uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } + + } array set ::incompatible [list] @@ -381,6 +393,48 @@ ifcapable fts3 { } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } + + # Now test that an incremental merge can be started by one version + # and finished by another. And that the integrity-check still + # passes. + do_test backcompat-3.8 { + sql1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE t2 USING fts3(words); + } + code1 [list source $testdir/genesis.tcl] + code1 { fts_kjv_genesis } + sql1 { + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1 2 3 4 5}} + + if {[code1 { set ::sqlite_libversion }] >=3071200 + && [code2 { set ::sqlite_libversion }] >=3071200 + } { + do_test backcompat-3.9 { + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); } + sql2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1} 1 0} + + do_test backcompat-3.10 { + sql1 { INSERT INTO t2(t2) VALUES('integrity-check') } + sql2 { INSERT INTO t2(t2) VALUES('integrity-check') } + } {} + } } } } From 77a90ce811107805c414f381285d5a818e5e34ea Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 16 May 2014 23:15:50 +0000 Subject: [PATCH 09/11] Work around compilation issue with MSVC. FossilOrigin-Name: 9623a29c11e3a90513016ea3306c085d2bf91b62 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 7 ++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 91c09ac09c..361e70cb17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sprevent\san\sFTS\sindex\sfrom\sgrowing\sindefinitely\sas\sthe\scorresponding\stable\sis\supdated.\sChange\sthe\sFTS\s'automerge'\soption\sto\sallow\sthe\suser\sto\sspecify\sthe\snumber\sof\ssegments\sthat\sshould\sbe\smerged\ssimultaneously\sby\sauto-merges. -D 2014-05-16T16:16:59.006 +C Work\saround\scompilation\sissue\swith\sMSVC. +D 2014-05-16T23:15:50.277 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c bf60d71edc33a48a6e4f7494da64aaf15e6542fd +F src/main.c 1c164c7b742f0cd60290b34dc9086df9f939f490 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 3997d47bb9c8593a2cd30317ea65163b99db1832 -R 01618b664df3c3b19706db5d70667d8a -U dan -Z fa66c3fc7c4b02244be8e29ed55dc1c5 +P a75f1800021d30b712053373440041b7b355a36a +R e44c4f2d8b2a74316856f263d26bcc16 +U mistachkin +Z 7ea453b006be693b9823cb9b81301836 diff --git a/manifest.uuid b/manifest.uuid index d4cefb8a54..92cf7444c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a75f1800021d30b712053373440041b7b355a36a \ No newline at end of file +9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file diff --git a/src/main.c b/src/main.c index b3bc66e849..d150d19d3c 100644 --- a/src/main.c +++ b/src/main.c @@ -3126,7 +3126,12 @@ int sqlite3_test_control(int op, ...){ ** sqlite3_test_control(). */ case SQLITE_TESTCTRL_FAULT_INSTALL: { - sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3Config.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3Config.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); rc = sqlite3FaultSim(0); break; } From 02c1e264a9fa144181b3898e010ee1050b6658b9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 16 May 2014 23:38:59 +0000 Subject: [PATCH 10/11] Fix harmless compiler warnings. FossilOrigin-Name: 7e60347e21a7473974061965a8d1f2eadde987e2 --- ext/fts3/fts3_write.c | 12 ++++++------ manifest | 15 +++++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 2cbf084e84..ea3c08802a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3044,7 +3044,7 @@ static void fts3ReadEndBlockField( */ static int fts3PromoteSegments( Fts3Table *p, /* FTS table handle */ - int iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ ){ int rc = SQLITE_OK; @@ -3054,7 +3054,7 @@ static int fts3PromoteSegments( if( rc==SQLITE_OK ){ int bOk = 0; - int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; i64 nLimit = (nByte*3)/2; /* Loop through all entries in the %_segdir table corresponding to @@ -3062,8 +3062,8 @@ static int fts3PromoteSegments( ** at least one such segment, and it is possible to determine that all ** such segments are smaller than nLimit bytes in size, they will be ** promoted to level iAbsLevel. */ - sqlite3_bind_int(pRange, 1, iAbsLevel+1); - sqlite3_bind_int(pRange, 2, iLast); + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); @@ -3103,7 +3103,7 @@ static int fts3PromoteSegments( ** setting the "idx" fields as appropriate to keep them in the same ** order. The contents of level -1 (which is never used, except ** transiently here), will be moved back to level iAbsLevel below. */ - sqlite3_bind_int(pRange, 1, iAbsLevel); + sqlite3_bind_int64(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); @@ -3122,7 +3122,7 @@ static int fts3PromoteSegments( /* Move level -1 to level iAbsLevel */ if( rc==SQLITE_OK ){ - sqlite3_bind_int(pUpdate2, 1, iAbsLevel); + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); rc = sqlite3_reset(pUpdate2); } diff --git a/manifest b/manifest index 361e70cb17..0cdba0d21b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\scompilation\sissue\swith\sMSVC. -D 2014-05-16T23:15:50.277 +C Fix\sharmless\scompiler\swarnings. +D 2014-05-16T23:38:59.727 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 6681e4855bd24e74e91e2017f3fdc44460449fc1 +F ext/fts3/fts3_write.c 9e4e8579ad26cff3ee7743aaf5c3fe937fc441b4 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -1172,7 +1172,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a75f1800021d30b712053373440041b7b355a36a -R e44c4f2d8b2a74316856f263d26bcc16 +P 9623a29c11e3a90513016ea3306c085d2bf91b62 +R 5ded43b85153ff3080cc0993c5b9bd75 +T *branch * ftsWarnings +T *sym-ftsWarnings * +T -sym-trunk * U mistachkin -Z 7ea453b006be693b9823cb9b81301836 +Z 3f5cc6ff89fc3bbade4957ecc9f7a199 diff --git a/manifest.uuid b/manifest.uuid index 92cf7444c6..7ca9a7ea7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file +7e60347e21a7473974061965a8d1f2eadde987e2 \ No newline at end of file From 83e05b814eb8a625fdb6ee6d3b630408d3dd19d5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 May 2014 15:10:37 +0000 Subject: [PATCH 11/11] Fix a broken test case in fts4growth2.test. FossilOrigin-Name: 9fde5b960af39b187474e5b2909ff0e56d5e0bf9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts4growth2.test | 11 ++++++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 361e70cb17..48cfe8e5aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\scompilation\sissue\swith\sMSVC. -D 2014-05-16T23:15:50.277 +C Fix\sa\sbroken\stest\scase\sin\sfts4growth2.test. +D 2014-05-17T15:10:37.982 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -571,7 +571,7 @@ F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test df10fde9f47cf5c71861e63fd8efcd573c4f7e53 -F test/fts4growth2.test 4ea35ea97c7d87e8f19d5ede939ab356f9f75cf0 +F test/fts4growth2.test 2f063be1902a73cd087355837c52fed42ac11a5d F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9 F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee @@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a75f1800021d30b712053373440041b7b355a36a -R e44c4f2d8b2a74316856f263d26bcc16 -U mistachkin -Z 7ea453b006be693b9823cb9b81301836 +P 9623a29c11e3a90513016ea3306c085d2bf91b62 +R 7fbf0d1c98f8f088bba70604a90180b5 +U dan +Z 9d3390acf23fdcf50586fd0f9fba7a93 diff --git a/manifest.uuid b/manifest.uuid index 92cf7444c6..99244e27d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9623a29c11e3a90513016ea3306c085d2bf91b62 \ No newline at end of file +9fde5b960af39b187474e5b2909ff0e56d5e0bf9 \ No newline at end of file diff --git a/test/fts4growth2.test b/test/fts4growth2.test index 023ca8ba1c..af41d51c18 100644 --- a/test/fts4growth2.test +++ b/test/fts4growth2.test @@ -28,14 +28,19 @@ source $testdir/genesis.tcl do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } fts_kjv_genesis +proc structure {} { + puts [ db eval {SELECT level, count(*) FROM x1_segdir GROUP BY level} ] +} + proc tt {val} { execsql { DELETE FROM x1 WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + } + execsql { INSERT INTO x1(docid, content) SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; } - #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] } do_execsql_test 1.1 { @@ -49,9 +54,9 @@ do_test 1.2 { } execsql { SELECT max(level) FROM x1_segdir; - SELECT count(*) FROM x1_segdir WHERE level=3; + SELECT count(*) FROM x1_segdir WHERE level=2; } -} {3 1} +} {2 1} do_test 1.3 { for {set i 0} {$i < 40} {incr i} {