From 2938f924cc34afaff7a5a11dad621046ea9b0c1b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 Mar 2012 19:13:29 +0000 Subject: [PATCH 01/68] If a CHECK constraint is named, report that name on the error message when the constraint fails. FossilOrigin-Name: 9a0f90d9deb335ac71044b8afa81538d85cc7ccf --- manifest | 25 ++++++++++++++----------- manifest.uuid | 2 +- src/build.c | 19 +++++++++++++------ src/insert.c | 40 ++++++++++++++++++++++++++-------------- src/parse.y | 6 +++--- src/sqliteInt.h | 3 ++- test/check.test | 12 ++++++------ 7 files changed, 65 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 804df3f20b..dc2ab319db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scompiling\sthe\stest\scode\sin\sfts3_test.c\swhen\sSQLITE_ENABLE_FTS3\sis\snot\sdefined. -D 2012-03-05T16:24:26.279 +C If\sa\sCHECK\sconstraint\sis\snamed,\sreport\sthat\sname\son\sthe\serror\smessage\swhen\nthe\sconstraint\sfails. +D 2012-03-07T19:13:29.390 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -128,7 +128,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 253c3147a4ebbaee42cd329dbdc0856200bbbda7 F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c -F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 +F src/build.c d58d314c5837737e15f94959cb55115347725182 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 @@ -142,7 +142,7 @@ F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d7c69718acbb92e10e4b121da7bed13903342962 +F src/insert.c 82b1bc7aaa54d0f058101a034a28d47fe6b70613 F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 @@ -170,7 +170,7 @@ F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 3955b62cdb5bb64559607cb474dd12a6c8e1d4a5 F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 -F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e +F src/parse.y 180976f414ba0473bffa7049b37aceab1e467175 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 @@ -184,7 +184,7 @@ F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e F src/sqlite.h.in f46e368d1a28b09d876e35444785674d170f2d62 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h b013dab7d43fb67c3ca2f0253d7863abb37e233c +F src/sqliteInt.h 306a6f0c6732c0d706dfebfc780a133a38507aee F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -320,7 +320,7 @@ F test/capi3c.test 1b5424d2ac57b7b443b5de5b9a287642c02279b6 F test/capi3d.test 17b57ca28be3e37e14c2ba8f787d292d84b724a1 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/check.test db2b29d557544347d28e25b8406f5d5ecc3d1bc3 +F test/check.test 06795c188bf1776673fd2ac0787aa1c7238970d8 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 @@ -992,7 +992,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 99a9073b5e411ce94f38ce49608baaa15de8b850 -R d0dc4635c0ac977f7342c0a0dac9e48b -U dan -Z d6573fd4e75425ab6ee13bee3b518e09 +P b00ccda307caae597c143ab0586f90acb77f79cf +R 5242d5c6432e029c0fd352c1dbfa159a +T *branch * named-check-constraints +T *sym-named-check-constraints * +T -sym-trunk * +U drh +Z 198934e577031b93f87585ac1616330d diff --git a/manifest.uuid b/manifest.uuid index dda6c5d414..65912e34e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b00ccda307caae597c143ab0586f90acb77f79cf \ No newline at end of file +9a0f90d9deb335ac71044b8afa81538d85cc7ccf \ No newline at end of file diff --git a/src/build.c b/src/build.c index daa5430406..16a9978fcf 100644 --- a/src/build.c +++ b/src/build.c @@ -537,7 +537,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); + sqlite3ExprListDelete(db, pTable->pCheck); #endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); @@ -1200,15 +1200,17 @@ void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ - sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } }else #endif { - sqlite3ExprDelete(db, pCheckExpr); + sqlite3ExprDelete(pParse->db, pCheckExpr); } } @@ -1478,6 +1480,8 @@ void sqlite3EndTable( if( p->pCheck ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + ExprList *pList; /* List of all CHECK constraints */ + int i; /* Loop counter */ memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); @@ -1488,8 +1492,11 @@ void sqlite3EndTable( sNC.pParse = pParse; sNC.pSrcList = &sSrc; sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; + pList = p->pCheck; + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } } } #endif /* !defined(SQLITE_OMIT_CHECK) */ diff --git a/src/insert.c b/src/insert.c index 6b31e24f2c..18ed8cfcf3 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1157,9 +1157,11 @@ void sqlite3GenerateConstraintChecks( int regData; /* Register containing first data column */ int iCur; /* Table cursor number */ Index *pIdx; /* Pointer to one of the indices */ + sqlite3 *db; /* Database connection */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; + db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1192,7 +1194,7 @@ void sqlite3GenerateConstraintChecks( char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); break; @@ -1214,18 +1216,28 @@ void sqlite3GenerateConstraintChecks( /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + int i; pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); + for(i=0; inExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zConsName = pCheck->a[i].zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( zConsName ){ + zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + }else{ + zConsName = sqlite3MPrintf(db, "constraint failed"); + } + sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + } + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -1281,7 +1293,7 @@ void sqlite3GenerateConstraintChecks( ** table. */ Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ @@ -1370,7 +1382,7 @@ void sqlite3GenerateConstraintChecks( char *zErr; sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; + errMsg.db = db; zSep = pIdx->nColumn>1 ? "columns " : "column "; for(j=0; jnColumn; j++){ char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; @@ -1394,7 +1406,7 @@ void sqlite3GenerateConstraintChecks( Trigger *pTrigger = 0; assert( onError==OE_Replace ); sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete( @@ -1724,7 +1736,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck, pDest->pCheck) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/src/parse.y b/src/parse.y index ed18e7f973..29b58a43df 100644 --- a/src/parse.y +++ b/src/parse.y @@ -273,10 +273,10 @@ signed ::= minus_num. // "carglist" is a list of additional constraints that come after the // column name and column type in a CREATE TABLE statement. // -carglist ::= carglist carg. +carglist ::= carglist cname ccons. carglist ::= . -carg ::= CONSTRAINT nm ccons. -carg ::= ccons. +cname ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +cname ::= . {pParse->constraintName.n = 0;} ccons ::= DEFAULT term(X). {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT LP expr(X) RP. {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT PLUS term(X). {sqlite3AddDefaultValue(pParse,&X);} diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cfe8fd64fb..dcc2ecec9a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1274,7 +1274,7 @@ struct Table { FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK - Expr *pCheck; /* The AND of all CHECK constraints */ + ExprList *pCheck; /* All CHECK constraints */ #endif #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ @@ -2208,6 +2208,7 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ + Token constraintName;/* Name of the constraint currently being parsed */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ diff --git a/test/check.test b/test/check.test index d2867a096e..4d1a44117d 100644 --- a/test/check.test +++ b/test/check.test @@ -117,9 +117,9 @@ do_test check-1.17 { do_test check-2.1 { execsql { CREATE TABLE t2( - x INTEGER CHECK( typeof(coalesce(x,0))=="integer" ), - y REAL CHECK( typeof(coalesce(y,0.1))=='real' ), - z TEXT CHECK( typeof(coalesce(z,''))=='text' ) + x INTEGER CONSTRAINT one CHECK( typeof(coalesce(x,0))=="integer" ), + y REAL CONSTRAINT two CHECK( typeof(coalesce(y,0.1))=='real' ), + z TEXT CONSTRAINT three CHECK( typeof(coalesce(z,''))=='text' ) ); } } {} @@ -141,17 +141,17 @@ do_test check-2.4 { catchsql { INSERT INTO t2 VALUES(1.1, NULL, NULL); } -} {1 {constraint failed}} +} {1 {constraint one failed}} do_test check-2.5 { catchsql { INSERT INTO t2 VALUES(NULL, 5, NULL); } -} {1 {constraint failed}} +} {1 {constraint two failed}} do_test check-2.6 { catchsql { INSERT INTO t2 VALUES(NULL, NULL, 3.14159); } -} {1 {constraint failed}} +} {1 {constraint three failed}} ifcapable subquery { do_test check-3.1 { From 593c9824bcb36dce3a8d1a5b939e9b4be41f6596 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Mar 2012 18:39:03 +0000 Subject: [PATCH 02/68] =?UTF-8?q?Add=20the=20'merge=3D=3F,=3F'=20command?= =?UTF-8?q?=20to=20fts4.=20This=20still=20needs=20some=20work.?= FossilOrigin-Name: 741b8f897750eac3c9774fd65de7e40bb89781b1 --- ext/fts3/fts3Int.h | 5 +- ext/fts3/fts3_write.c | 1121 ++++++++++++++++++++++++++++++++++++++++- manifest | 20 +- manifest.uuid | 2 +- test/fts3_common.tcl | 9 +- test/fts4merge.test | 103 ++++ 6 files changed, 1232 insertions(+), 28 deletions(-) create mode 100644 test/fts4merge.test diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 393cd6aea1..b34ca63052 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -67,6 +67,9 @@ extern const sqlite3_api_routines *sqlite3_api; #ifndef MIN # define MIN(x,y) ((x)<(y)?(x):(y)) #endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif /* ** Maximum length of a varint encoded integer. The varint format is different @@ -197,7 +200,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[28]; + sqlite3_stmt *aStmt[35]; char *zReadExprlist; char *zWriteExprlist; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 8f97c8be98..8f3b465181 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -24,6 +24,9 @@ #include #include + +#define FTS_MAX_APPENDABLE_HEIGHT 10 + /* ** When full-text index nodes are loaded from disk, the buffer that they ** are loaded into has the following number of bytes of padding at the end @@ -229,10 +232,15 @@ struct SegmentNode { #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 - #define SQL_DELETE_SEGDIR_RANGE 26 - #define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRIES 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 /* ** This function is used to obtain an SQLite prepared statement handle @@ -261,9 +269,9 @@ static int fts3SqlStmt( /* 6 */ "DELETE FROM %Q.'%q_stat'", /* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " @@ -289,6 +297,45 @@ static int fts3SqlStmt( /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>?" + " ORDER BY (level %% 1024) DESC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the two segments with idx=0 and idx=1 from absolute +** level ?. See function fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx BETWEEN 0 AND 1", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRIES +** Reduce by one the idx values of all segments on absolute level :1 with +** an index greater than :2. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = idx - 1 WHERE level = ? AND idx>:2", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -446,6 +493,24 @@ static sqlite3_int64 getAbsoluteLevel( return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; } +/* +** Given an absolute level number, determine the langauge-id, index +** and relative level that it corresponds to. +** +** The return value is the relative level. The language-id and index +** are returned via output variables. +*/ +static int getRelativeLevel( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level */ + int *piLangid, /* OUT: Language id */ + int *piIndex /* OUT: Index in p->aIndex[] */ +){ + if( piLangid ) *piLangid = (iAbsLevel / FTS3_SEGDIR_MAXLEVEL) / p->nIndex; + if( piIndex ) *piIndex = (iAbsLevel / FTS3_SEGDIR_MAXLEVEL) % p->nIndex; + return iAbsLevel % FTS3_SEGDIR_MAXLEVEL; +} + /* ** Set *ppStmt to a statement handle that may be used to iterate through @@ -1059,7 +1124,7 @@ int sqlite3Fts3ReadBlock( int rc; /* Return code */ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob); + assert( pnBlob ); if( p->pSegments ){ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); @@ -2263,6 +2328,29 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + /* ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, @@ -2285,19 +2373,12 @@ static int fts3DeleteSegdir( Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ int i; /* Iterator variable */ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); for(i=0; rc==SQLITE_OK && iiStartBlock ){ - sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } + rc = fts3DeleteSegment(p, apSegment[i]); } if( rc!=SQLITE_OK ){ return rc; @@ -3142,6 +3223,1014 @@ static int fts3DoRebuild(Fts3Table *p){ return rc; } + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest two segments (idx=0 and idx=1) in absolute level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; + + memset(pCsr, 0, sizeof(*pCsr)); + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader *)*2); + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, sizeof(Fts3SegReader *)*2); + pCsr->nSegment = 2; + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<2; i++){ + rc = sqlite3Fts3SegReaderNew(i, 0, + sqlite3_column_int64(pStmt, 1), /* segdir.start_block */ + sqlite3_column_int64(pStmt, 2), /* segdir.leaves_end_block */ + sqlite3_column_int64(pStmt, 3), /* segdir.end_block */ + sqlite3_column_blob(pStmt, 4), /* segdir.root */ + sqlite3_column_bytes(pStmt, 4), /* segdir.root */ + &pCsr->apSegment[i] + ); + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct LayerWriter LayerWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +struct Blob { + char *a; + int n; + int nAlloc; +}; + +struct LayerWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + LayerWriter aLayer[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aLayer[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aLayer[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; iLayeraLayer[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pLayer->key.a, pLayer->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pLayer->key.n==0 || (pLayer->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pLayer->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pLayer->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pLayer->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pLayer->key.a, zTerm, nTerm); + pLayer->key.n = nTerm; + } + }else{ + /* Otherwise, flush the the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a,pLayer->block.n); + + assert( pLayer->block.nAlloc>=p->nNodeSize ); + pLayer->block.a[0] = (char)iLayer; + pLayer->block.n = 1 + sqlite3Fts3PutVarint(&pLayer->block.a[1], iPtr+1); + + iNextPtr = pLayer->iBlock; + pLayer->iBlock++; + pLayer->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); +} + +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; + int nSuffix; + LayerWriter *pLayer; + Blob *pPg; + + pLayer = &pWriter->aLayer[0]; + nPrefix = fts3PrefixCompress(pLayer->key.a, pLayer->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLayer->block.n>0 && (pLayer->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a, pLayer->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLayer->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLayer->iBlock++; + pLayer->key.n = 0; + pLayer->block.n = 0; + + nPrefix = 0; + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + blobGrowBuffer(&pLayer->key, nTerm, &rc); + blobGrowBuffer(&pLayer->block, pLayer->block.n + nSpace, &rc); + + if( rc==SQLITE_OK ){ + /* Update the block image with the new entry */ + pPg = &pLayer->block; + pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nPrefix); + pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nSuffix); + memcpy(&pPg->a[pPg->n], &zTerm[nPrefix], nSuffix); + pPg->n += nSuffix; + pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nDoclist); + memcpy(&pPg->a[pPg->n], aDoclist, nDoclist); + pPg->n += nDoclist; + + /* Take a copy of the key just written */ + memcpy(pLayer->key.a, zTerm, nTerm); + pLayer->key.n = nTerm; + } + + return rc; +} + +static void fts3IncrmergeRelease( + Fts3Table *p, + IncrmergeWriter *pWriter, + int *pRc +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; + LayerWriter *pRoot; + int rc = *pRc; + + /* Find the root node */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + if( pWriter->aLayer[iRoot].block.n>0 ) break; + assert( pWriter->aLayer[iRoot].block.nAlloc==0 ); + assert( pWriter->aLayer[iRoot].key.nAlloc==0 ); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on the root node. This is not allowed. */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aLayer[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aLayer[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aLayer[iRoot]; + + for(i=0; iaLayer[i]; + if( pLayer->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a,pLayer->block.n); + } + sqlite3_free(pLayer->block.a); + sqlite3_free(pLayer->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aLayer[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + + +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pRc){ + int bRes = 0; + if( *pRc==SQLITE_OK ){ + sqlite3_stmt *pCheck = 0; + int rc; + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + *pRc = rc; + } + + return bRes; +} + +/* +** +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + sqlite3_int64 iStart = 0; + sqlite3_int64 iLeafEnd = 0; + sqlite3_int64 iEnd = 0; + const char *aRoot = 0; + int nRoot = 0; + int rc; + + sqlite3_stmt *pSelect = 0; + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + int bAppendable = 0; + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + iEnd = sqlite3_column_int64(pSelect, 3); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + bAppendable = fts3IsAppendable(p, iEnd, &rc); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = (char *)aRoot; + int nLeaf = nRoot; + + if( aRoot[0] ){ + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + } + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + if( aLeaf!=aRoot ) sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + LayerWriter *pLayer; + + pWriter->nLeafEst = ((iEnd - iStart) + 1) / FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + pLayer = &pWriter->aLayer[nHeight]; + pLayer->iBlock = pWriter->iStart + nHeight*FTS_MAX_APPENDABLE_HEIGHT; + blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pLayer->block.a, aRoot, nRoot); + pLayer->block.n = nRoot; + } + + for(i=(int)aRoot[0]; i>=0 && rc==SQLITE_OK; i--){ + pLayer = &pWriter->aLayer[i]; + NodeReader reader; + + rc = nodeReaderInit(&reader, pLayer->block.a, pLayer->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pLayer->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pLayer->key.a, reader.term.a, reader.term.n); + pLayer->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pLayer = &pWriter->aLayer[i-1]; + pLayer->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pLayer->block, nBlock, &rc); + if( rc==SQLITE_OK ){ + memcpy(pLayer->block.a, aBlock, nBlock); + pLayer->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Either allocate an output segment or locate an existing appendable +** output segment to append to. And "appendable" output segment is +** slightly different to a normal one, as the required range of keys in +** the %_segments table must be allocated up front. This requires some +** assumptions: +** +** * It is expected that due to the short-keys used, and the prefix and +** suffix compression, the fanout of segment b-trees will be very high. +** With a conservative assumption of 32 bytes per key and 1024 byte +** pages, say 32 (2^5). Since SQLite database files are limited to +** a total of 2^31 pages in size, it seems very likely that no segment +** b-tree will have more than ten layers of nodes (including the +** leaves). +** +** * Since each interior node has a pointer to at least two child nodes, +** each layer of interior nodes must be smaller than the layer of +** leaf nodes. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the two input segments multiplied +** by three. If an input segment consists of a root node only, treat it +** as if it has a single leaf node for the purposes of this estimate. +** This value is stored in stack variable nLeafEst. +** +** A total of 10*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==10*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 10). And so on. +** +** In the actual code below, the value "10" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst; /* Blocks allocated for leaf nodes */ + int iIdx; /* Index of output segment */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + iIdx = sqlite3_column_int(pOutputIdx, 0) - 1; + rc = sqlite3_reset(pOutputIdx); + } + if( rc!=SQLITE_OK ) return rc; + + assert( zKey ); + assert( pWriter->nLeafEst==0 ); + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx, zKey, nKey, pWriter); + if( rc!=SQLITE_OK || pWriter->nLeafEst ) return rc; + iIdx++; + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + } + + /* Set up the array of LayerWriter objects */ + for(i=0; iaLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, + sqlite3_int64 iAbsLevel, + int iIdx +){ + int rc; + sqlite3_stmt *pDelete = 0; + sqlite3_stmt *pUpdate = 0; + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRIES, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 1, iAbsLevel); + sqlite3_bind_int(pUpdate, 2, iIdx); + sqlite3_step(pUpdate); + rc = sqlite3_reset(pUpdate); + } + + return rc; +} + +static int fts3AppendToNode( + Blob *pNode, + Blob *pPrev, + const char *zTerm, + int nTerm, + const char *aDoclist, + int nDoclist +){ + int rc = SQLITE_OK; + int bFirst = (pPrev->n==0); + int nPrefix; + int nSuffix; + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + return SQLITE_OK; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; + int rc = SQLITE_OK; + int bStarted = 0; + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( bStarted==0 ){ + if( fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm)<0 ) continue; + pNew->a[0] = aNode[0]; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + bStarted = 1; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + } + if( bStarted==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; + sqlite3_int64 iOldStart = 0; + int rc2; /* sqlite3_reset() return code */ + + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, + sqlite3_int64 iAbsLevel, + Fts3MultiSegReader *pCsr +){ + int i; + int rc = SQLITE_OK; + + assert( pCsr->nSegment==2 ); + assert( (pCsr->apSegment[0]->iIdx==0 && pCsr->apSegment[1]->iIdx==1) + || (pCsr->apSegment[1]->iIdx==0 && pCsr->apSegment[0]->iIdx==1) + ); + + for(i=1; i>=0; i--){ + Fts3SegReader *pSeg = pCsr->apSegment[0]; + if( pSeg->iIdx!=i ) pSeg = pCsr->apSegment[1]; + assert( pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + } + } + + return rc; +} + +static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc = SQLITE_OK; /* Return code */ + int nRem = nMerge; + + assert( nMin>=2 ); + + while( rc==SQLITE_OK && nRem>0 ){ + sqlite3_int64 iAbsLevel; /* Absolute level number to work on */ + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + Fts3MultiSegReader csr; /* Cursor used to read input data */ + Fts3SegFilter filter; /* Filter used with cursor csr */ + IncrmergeWriter writer; /* Writer object */ + + memset(&writer, 0, sizeof(IncrmergeWriter)); + + /* Determine which level to merge segments from. Any level, from any + ** prefix or language index may be selected. Stack variable iAbsLevel + ** is set to the absolute level number of the level to merge from. */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ + return sqlite3_reset(pFindLevel); + } + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + rc = sqlite3_reset(pFindLevel); + if( rc!=SQLITE_OK ) return rc; + + /* Open a cursor to iterate through the contents of indexes 0 and 1 of + ** the selected absolute level. */ + rc = fts3IncrmergeCsr(p, iAbsLevel, &csr); + if( rc!=SQLITE_OK ) return rc; + memset(&filter, 0, sizeof(Fts3SegFilter)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS; + + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + assert( rc!=SQLITE_ABORT ); + if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer); + assert( rc!=SQLITE_ABORT ); + if( rc==SQLITE_OK ){ + do { + rc = fts3IncrmergeAppend(p, &writer, &csr); + assert( rc!=SQLITE_ABORT ); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr); + assert( rc!=SQLITE_ABORT ); + if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } + } + assert( rc!=SQLITE_ABORT ); + fts3IncrmergeRelease(p, &writer, &rc); + nRem -= (1 + writer.nWork); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeChomp(p, iAbsLevel, &csr); + } + + sqlite3Fts3SegReaderFinish(&csr); + } + + return rc; +} + +static int fts3_isdigit(char c){ + return (c>='0' && c<='9'); +} + +static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + for(z=zParam; fts3_isdigit(z[0]); z++){ + nMerge = nMerge * 10 + (z[0] - '0'); + } + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = 0; + while( fts3_isdigit(z[0]) ){ + nMin = nMin * 10 + (z[0] - '0'); + z++; + } + } + + if( z[0]!='\0' ) return SQLITE_ERROR; + if( nMin<2 ) nMin = 2; + + rc = fts3Incrmerge(p, nMerge, nMin); + sqlite3Fts3SegmentsClose(p); + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -3161,6 +4250,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoOptimize(p, 0); }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ rc = fts3DoRebuild(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); diff --git a/manifest b/manifest index 804df3f20b..b84b66edfd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scompiling\sthe\stest\scode\sin\sfts3_test.c\swhen\sSQLITE_ENABLE_FTS3\sis\snot\sdefined. -D 2012-03-05T16:24:26.279 +C Add\sthe\s'merge=?,?'\scommand\sto\sfts4.\sThis\sstill\sneeds\ssome\swork. +D 2012-03-08T18:39:03.077 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -65,7 +65,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 806632fd0020eed966ab82ea25fe09f1a4c86907 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h d1d7f964ddee067bcd16a6af4ba7ecf66220056d +F ext/fts3/fts3Int.h cc8991daf660b926ef77a004fe92b97adaf10d97 F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073 +F ext/fts3/fts3_write.c 81b885ef5db55c2f92b8e5a33e51eb7557084e41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -448,7 +448,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 -F test/fts3_common.tcl 4d8eec9db565fed9098f45c378f28e1657802011 +F test/fts3_common.tcl 91c29230c428443e6552add9b18cf94a3dc23074 F test/fts3aa.test 909d5f530d30a8e36b9328d67285eae6537c79c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 @@ -497,6 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a +F test/fts4merge.test 3af8fa8fd9f27a9eb402b9ae399aa53fbaae8481 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,7 +993,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 99a9073b5e411ce94f38ce49608baaa15de8b850 -R d0dc4635c0ac977f7342c0a0dac9e48b +P b00ccda307caae597c143ab0586f90acb77f79cf +R a895af2ffe98328624a86f767eb1318f +T *branch * fts4-incr-merge +T *sym-fts4-incr-merge * +T -sym-trunk * U dan -Z d6573fd4e75425ab6ee13bee3b518e09 +Z 1441d373e5b74aab517b4d814807ebb3 diff --git a/manifest.uuid b/manifest.uuid index dda6c5d414..b25709a4f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b00ccda307caae597c143ab0586f90acb77f79cf \ No newline at end of file +741b8f897750eac3c9774fd65de7e40bb89781b1 \ No newline at end of file diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index 4d4ae38bae..23a8cfca67 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -46,6 +46,7 @@ proc fts3_integrity_check {tbl} { fts3_read2 $tbl 1 A foreach zTerm [array names A] { + #puts $zTerm foreach doclist $A($zTerm) { set docid 0 while {[string length $doclist]>0} { @@ -233,7 +234,8 @@ proc fts3_readleaf {blob} { set zTerm [string range $zPrev 0 [expr $nPrefix-1]] append zTerm [gobble_string blob $nSuffix] - set doclist [gobble_string blob [gobble_varint blob]] + set nDoclist [gobble_varint blob] + set doclist [gobble_string blob $nDoclist] lappend terms $zTerm $doclist set zPrev $zTerm @@ -249,7 +251,9 @@ proc fts3_read2 {tbl where varname} { FROM ${tbl}_segdir WHERE $where ORDER BY level ASC, idx DESC " { - if {$start_block == 0} { + set c 0 + binary scan $root c c + if {$c==0} { foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } } else { db eval " SELECT block @@ -258,7 +262,6 @@ proc fts3_read2 {tbl where varname} { ORDER BY blockid " { foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } - } } } diff --git a/test/fts4merge.test b/test/fts4merge.test new file mode 100644 index 0000000000..805a78cf97 --- /dev/null +++ b/test/fts4merge.test @@ -0,0 +1,103 @@ +# 2012 March 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the incremental merge function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4merge + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +proc fts3_build_db_1 {n} { + + if {$n > 10000} {error "n must be <= 10000"} + + db eval { CREATE VIRTUAL TABLE t1 USING fts4(x, y) } + + set xwords [list zero one two three four five six seven eight nine ten] + set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] + + for {set i 0} {$i < $n} {incr i} { + set x "" + set y "" + + set x [list] + lappend x [lindex $xwords [expr ($i / 1000) % 10]] + lappend x [lindex $xwords [expr ($i / 100) % 10]] + lappend x [lindex $xwords [expr ($i / 10) % 10]] + lappend x [lindex $xwords [expr ($i / 1) % 10]] + + set y [list] + lappend y [lindex $ywords [expr ($i / 1000) % 10]] + lappend y [lindex $ywords [expr ($i / 100) % 10]] + lappend y [lindex $ywords [expr ($i / 10) % 10]] + lappend y [lindex $ywords [expr ($i / 1) % 10]] + + db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) } + } +} + +#------------------------------------------------------------------------- +# Test cases 1.* +# +do_test 1.0 { fts3_build_db_1 1004 } {} +do_test 1.1 { fts3_integrity_check t1 } {ok} +do_execsql_test 1.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} + +for {set i 0} {$i<20} {incr i} { + do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } + do_test 1.2.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.2.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} +} + +do_execsql_test 1.3 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7} + 2 {0 1 2 3 4 5 6} +} + +for {set i 0} {$i<100} {incr i} { + do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } + do_test 1.4.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.4.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} +} + +do_execsql_test 1.5 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3} + 1 {0 1 2 3} + 2 {0 1 2 3} + 3 {0 1 2} +} + + +finish_test + From ab4f6385de75fdc97033eb77b6f79c21e6c10dfe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 9 Mar 2012 12:52:43 +0000 Subject: [PATCH 03/68] Minor commenting and stylistic changes only. FossilOrigin-Name: a1747086c5e0c152fcf4bd9fa80a61b6f03f4a94 --- ext/fts3/fts3_write.c | 36 +++++++++++++++++++++++++----------- manifest | 17 +++++++---------- manifest.uuid | 2 +- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 8f3b465181..302f284897 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4132,9 +4132,18 @@ static int fts3IncrmergeChomp( return rc; } +/* +** Attempt an incremental merge that writes nMerge leaf pages. +** +** Incremental merges happen two segments at a time. The two +** segments to be merged are the two oldest segments (the ones with +** the smallest index) in the highest level that has at least +** nMin segments. Multiple segment pair merges might occur in +** an attempt to write the quota of nMerge leaf pages. +*/ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc = SQLITE_OK; /* Return code */ - int nRem = nMerge; + int nRem = nMerge; /* Number of leaf pages yet to be written */ assert( nMin>=2 ); @@ -4161,7 +4170,7 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc!=SQLITE_OK ) return rc; /* Open a cursor to iterate through the contents of indexes 0 and 1 of - ** the selected absolute level. */ + ** the selected absolute level. */ rc = fts3IncrmergeCsr(p, iAbsLevel, &csr); if( rc!=SQLITE_OK ) return rc; memset(&filter, 0, sizeof(Fts3SegFilter)); @@ -4171,13 +4180,13 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ assert( rc!=SQLITE_ABORT ); if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer); - assert( rc!=SQLITE_ABORT ); + assert( rc!=SQLITE_ABORT ); if( rc==SQLITE_OK ){ do { rc = fts3IncrmergeAppend(p, &writer, &csr); - assert( rc!=SQLITE_ABORT ); + assert( rc!=SQLITE_ABORT ); if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr); - assert( rc!=SQLITE_ABORT ); + assert( rc!=SQLITE_ABORT ); if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; }while( rc==SQLITE_ROW ); } @@ -4197,10 +4206,15 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ return rc; } -static int fts3_isdigit(char c){ - return (c>='0' && c<='9'); -} - +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ int rc; int nMin = (FTS3_MERGE_COUNT / 2); @@ -4208,7 +4222,7 @@ static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ const char *z = zParam; /* Read the first integer value */ - for(z=zParam; fts3_isdigit(z[0]); z++){ + for(z=zParam; z[0]>='0' && z[0]<='9'; z++){ nMerge = nMerge * 10 + (z[0] - '0'); } @@ -4217,7 +4231,7 @@ static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ if( z[0]==',' && z[1]!='\0' ){ z++; nMin = 0; - while( fts3_isdigit(z[0]) ){ + while( z[0]>='0' && z[0]<='9' ){ nMin = nMin * 10 + (z[0] - '0'); z++; } diff --git a/manifest b/manifest index b84b66edfd..4eabdd282c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s'merge=?,?'\scommand\sto\sfts4.\sThis\sstill\sneeds\ssome\swork. -D 2012-03-08T18:39:03.077 +C Minor\scommenting\sand\sstylistic\schanges\sonly. +D 2012-03-09T12:52:43.195 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 81b885ef5db55c2f92b8e5a33e51eb7557084e41 +F ext/fts3/fts3_write.c 80863b8ef7849c8c225071258e13326f30bf00f3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -993,10 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P b00ccda307caae597c143ab0586f90acb77f79cf -R a895af2ffe98328624a86f767eb1318f -T *branch * fts4-incr-merge -T *sym-fts4-incr-merge * -T -sym-trunk * -U dan -Z 1441d373e5b74aab517b4d814807ebb3 +P 741b8f897750eac3c9774fd65de7e40bb89781b1 +R ef47488babfef36b031a8c9c8f291bcb +U drh +Z 0b165accece5454f4134f86269851728 diff --git a/manifest.uuid b/manifest.uuid index b25709a4f0..046cc5983d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -741b8f897750eac3c9774fd65de7e40bb89781b1 \ No newline at end of file +a1747086c5e0c152fcf4bd9fa80a61b6f03f4a94 \ No newline at end of file From 604f58a686f0622295eaba8135fd05ba902c7acb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Mar 2012 19:56:34 +0000 Subject: [PATCH 04/68] Fix some bugs in the incremental merge code. Some remain. FossilOrigin-Name: bff21683705a61b8b8672e0b44c287d1dc7c32a9 --- ext/fts3/fts3_write.c | 32 ++++++++++++++++++++++---------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 302f284897..c7f61219b5 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3721,15 +3721,20 @@ static int fts3IncrmergeLoad( pWriter->iAbsLevel = iAbsLevel; pWriter->iIdx = iIdx; + for(i=nHeight+1; iaLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + pLayer = &pWriter->aLayer[nHeight]; - pLayer->iBlock = pWriter->iStart + nHeight*FTS_MAX_APPENDABLE_HEIGHT; + pLayer->iBlock = pWriter->iStart; + pLayer->iBlock += pWriter->nLeafEst*FTS_MAX_APPENDABLE_HEIGHT; blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc); if( rc==SQLITE_OK ){ memcpy(pLayer->block.a, aRoot, nRoot); pLayer->block.n = nRoot; } - for(i=(int)aRoot[0]; i>=0 && rc==SQLITE_OK; i--){ + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ pLayer = &pWriter->aLayer[i]; NodeReader reader; @@ -3866,14 +3871,14 @@ static int fts3IncrmergeWriter( rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); } + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + /* Set up the array of LayerWriter objects */ for(i=0; iaLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } - - pWriter->iAbsLevel = iAbsLevel; - pWriter->nLeafEst = nLeafEst; - pWriter->iIdx = iIdx; return SQLITE_OK; } @@ -3962,6 +3967,14 @@ static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ } } +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ static int fts3TruncateNode( const char *aNode, /* Current node image */ int nNode, /* Size of aNode in bytes */ @@ -3974,6 +3987,7 @@ static int fts3TruncateNode( Blob prev = {0, 0, 0}; int rc = SQLITE_OK; int bStarted = 0; + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ /* Allocate required output space */ blobGrowBuffer(pNew, nNode, &rc); @@ -3986,7 +4000,8 @@ static int fts3TruncateNode( rc = nodeReaderNext(&reader) ){ if( bStarted==0 ){ - if( fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm)<0 ) continue; + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; pNew->a[0] = aNode[0]; fts3StartNode(pNew, (int)aNode[0], reader.iChild); *piBlock = reader.iChild; @@ -4180,13 +4195,10 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ assert( rc!=SQLITE_ABORT ); if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer); - assert( rc!=SQLITE_ABORT ); if( rc==SQLITE_OK ){ do { rc = fts3IncrmergeAppend(p, &writer, &csr); - assert( rc!=SQLITE_ABORT ); if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr); - assert( rc!=SQLITE_ABORT ); if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; }while( rc==SQLITE_ROW ); } diff --git a/manifest b/manifest index 4eabdd282c..2a7a5b68da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scommenting\sand\sstylistic\schanges\sonly. -D 2012-03-09T12:52:43.195 +C Fix\ssome\sbugs\sin\sthe\sincremental\smerge\scode.\sSome\sremain. +D 2012-03-13T19:56:34.883 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 80863b8ef7849c8c225071258e13326f30bf00f3 +F ext/fts3/fts3_write.c d26ef19833b4a5058f4bd0c8b9f00b3d197d0e28 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 741b8f897750eac3c9774fd65de7e40bb89781b1 -R ef47488babfef36b031a8c9c8f291bcb -U drh -Z 0b165accece5454f4134f86269851728 +P a1747086c5e0c152fcf4bd9fa80a61b6f03f4a94 +R 4dfb01ab70bf7b72368d08052c15c6ea +U dan +Z 2b44052e1e88ac3fc7ddfb07e6e262a4 diff --git a/manifest.uuid b/manifest.uuid index 046cc5983d..68a7e3aa67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1747086c5e0c152fcf4bd9fa80a61b6f03f4a94 \ No newline at end of file +bff21683705a61b8b8672e0b44c287d1dc7c32a9 \ No newline at end of file From 8af1fc721e11c1220c4d44d1dd3d94750956940f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Mar 2012 11:51:31 +0000 Subject: [PATCH 05/68] Fix another bug in the incremental merge code. FossilOrigin-Name: f97b12e0955c4c29f9c31a186d72d87f7407782e --- ext/fts3/fts3_write.c | 5 ++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index c7f61219b5..3f033361e5 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3726,8 +3726,7 @@ static int fts3IncrmergeLoad( } pLayer = &pWriter->aLayer[nHeight]; - pLayer->iBlock = pWriter->iStart; - pLayer->iBlock += pWriter->nLeafEst*FTS_MAX_APPENDABLE_HEIGHT; + pLayer->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc); if( rc==SQLITE_OK ){ memcpy(pLayer->block.a, aRoot, nRoot); @@ -3750,7 +3749,7 @@ static int fts3IncrmergeLoad( pLayer = &pWriter->aLayer[i-1]; pLayer->iBlock = reader.iChild; rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pLayer->block, nBlock, &rc); + blobGrowBuffer(&pLayer->block, MAX(nBlock, p->nNodeSize), &rc); if( rc==SQLITE_OK ){ memcpy(pLayer->block.a, aBlock, nBlock); pLayer->block.n = nBlock; diff --git a/manifest b/manifest index 2a7a5b68da..321984d2ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sbugs\sin\sthe\sincremental\smerge\scode.\sSome\sremain. -D 2012-03-13T19:56:34.883 +C Fix\sanother\sbug\sin\sthe\sincremental\smerge\scode. +D 2012-03-14T11:51:31.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c d26ef19833b4a5058f4bd0c8b9f00b3d197d0e28 +F ext/fts3/fts3_write.c 828b6395666e1e6fba0f762c14a3c5a3c074d7d1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a1747086c5e0c152fcf4bd9fa80a61b6f03f4a94 -R 4dfb01ab70bf7b72368d08052c15c6ea +P bff21683705a61b8b8672e0b44c287d1dc7c32a9 +R 9f60203a889705a2753ea10821ad5057 U dan -Z 2b44052e1e88ac3fc7ddfb07e6e262a4 +Z 0463fff5f4e61a05ca6743e1edc364a0 diff --git a/manifest.uuid b/manifest.uuid index 68a7e3aa67..f29dd9b13a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bff21683705a61b8b8672e0b44c287d1dc7c32a9 \ No newline at end of file +f97b12e0955c4c29f9c31a186d72d87f7407782e \ No newline at end of file From a2af0aecdb3ccf3dff03034b03cb00ea052dd2fe Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Mar 2012 12:17:40 +0000 Subject: [PATCH 06/68] Avoid allocating a large object on the stack in the incremental merge code. Use sqlite3_malloc() instead. FossilOrigin-Name: 36ae510de45be44efd34cff242d02fb21b7419ac --- ext/fts3/fts3_write.c | 56 +++++++++++++++++++++++-------------------- manifest | 12 +++++----- manifest.uuid | 2 +- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3f033361e5..35da283df3 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -25,7 +25,7 @@ #include -#define FTS_MAX_APPENDABLE_HEIGHT 10 +#define FTS_MAX_APPENDABLE_HEIGHT 16 /* ** When full-text index nodes are loaded from disk, the buffer that they @@ -4164,17 +4164,15 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ while( rc==SQLITE_OK && nRem>0 ){ sqlite3_int64 iAbsLevel; /* Absolute level number to work on */ sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ - Fts3MultiSegReader csr; /* Cursor used to read input data */ - Fts3SegFilter filter; /* Filter used with cursor csr */ - IncrmergeWriter writer; /* Writer object */ - - memset(&writer, 0, sizeof(IncrmergeWriter)); + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); /* Determine which level to merge segments from. Any level, from any ** prefix or language index may be selected. Stack variable iAbsLevel ** is set to the absolute level number of the level to merge from. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int(pFindLevel, 1, nMin); if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ return sqlite3_reset(pFindLevel); @@ -4183,35 +4181,41 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ rc = sqlite3_reset(pFindLevel); if( rc!=SQLITE_OK ) return rc; + /* Allocate space for the cursor, filter and writer objects */ + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, nAlloc); + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + /* Open a cursor to iterate through the contents of indexes 0 and 1 of ** the selected absolute level. */ - rc = fts3IncrmergeCsr(p, iAbsLevel, &csr); - if( rc!=SQLITE_OK ) return rc; - memset(&filter, 0, sizeof(Fts3SegFilter)); - filter.flags = FTS3_SEGMENT_REQUIRE_POS; + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + rc = fts3IncrmergeCsr(p, iAbsLevel, pCsr); - rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); - assert( rc!=SQLITE_ABORT ); - if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ - rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer); - if( rc==SQLITE_OK ){ - do { - rc = fts3IncrmergeAppend(p, &writer, &csr); - if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr); - if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; - }while( rc==SQLITE_ROW ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter); + if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ + rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr->zTerm,pCsr->nTerm,pWriter); + if( rc==SQLITE_OK ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } } } - assert( rc!=SQLITE_ABORT ); - fts3IncrmergeRelease(p, &writer, &rc); - nRem -= (1 + writer.nWork); + fts3IncrmergeRelease(p, pWriter, &rc); + nRem -= (1 + pWriter->nWork); /* Update or delete the input segments */ if( rc==SQLITE_OK ){ - rc = fts3IncrmergeChomp(p, iAbsLevel, &csr); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr); } - sqlite3Fts3SegReaderFinish(&csr); + sqlite3Fts3SegReaderFinish(pCsr); + sqlite3_free(pWriter); } return rc; diff --git a/manifest b/manifest index 321984d2ab..2ce91db0a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sbug\sin\sthe\sincremental\smerge\scode. -D 2012-03-14T11:51:31.020 +C Avoid\sallocating\sa\slarge\sobject\son\sthe\sstack\sin\sthe\sincremental\smerge\scode.\sUse\ssqlite3_malloc()\sinstead. +D 2012-03-14T12:17:40.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 828b6395666e1e6fba0f762c14a3c5a3c074d7d1 +F ext/fts3/fts3_write.c 772b8c32b93c60b85b60c635de2ff5b3f49fd779 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bff21683705a61b8b8672e0b44c287d1dc7c32a9 -R 9f60203a889705a2753ea10821ad5057 +P f97b12e0955c4c29f9c31a186d72d87f7407782e +R 86bb2bcb41aa5bde569f19117ae756eb U dan -Z 0463fff5f4e61a05ca6743e1edc364a0 +Z da9e601d088fda8ad57ce1fc3070902a diff --git a/manifest.uuid b/manifest.uuid index f29dd9b13a..137182d9b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f97b12e0955c4c29f9c31a186d72d87f7407782e \ No newline at end of file +36ae510de45be44efd34cff242d02fb21b7419ac \ No newline at end of file From 5730ef599ce6570a54d35d0abf3e7d17b933a4d8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Mar 2012 20:01:52 +0000 Subject: [PATCH 07/68] Add tests for incremental merge code. FossilOrigin-Name: 570473729d6561d81e6e5f8884fd18487008636e --- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 198 +++++++++++++++++++++-------------------- manifest | 21 ++--- manifest.uuid | 2 +- test/fts3_common.tcl | 63 ++++++++++++- test/fts4merge.test | 80 +++++++++++------ test/fts4merge2.test | 38 ++++++++ test/permutations.test | 2 +- 8 files changed, 268 insertions(+), 138 deletions(-) create mode 100644 test/fts4merge2.test diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index b34ca63052..1cd14d304a 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -124,7 +124,7 @@ extern const sqlite3_api_routines *sqlite3_api; # define NEVER(X) (0) #else # define ALWAYS(x) (x) -# define NEVER(X) (x) +# define NEVER(x) (x) #endif /* diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 35da283df3..08f07c10b2 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3272,10 +3272,16 @@ typedef struct LayerWriter LayerWriter; typedef struct Blob Blob; typedef struct NodeReader NodeReader; +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ struct Blob { - char *a; - int n; - int nAlloc; + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ }; struct LayerWriter { @@ -3405,7 +3411,7 @@ static int fts3IncrmergePush( int iLayer; assert( nTerm>0 ); - for(iLayer=1; iLayeraLayer[iLayer]; int rc = SQLITE_OK; @@ -3565,9 +3571,12 @@ static void fts3IncrmergeRelease( /* Find the root node */ for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ - if( pWriter->aLayer[iRoot].block.n>0 ) break; - assert( pWriter->aLayer[iRoot].block.nAlloc==0 ); - assert( pWriter->aLayer[iRoot].key.nAlloc==0 ); + LayerWriter *pLayer = &pWriter->aLayer[iRoot]; + if( pLayer->block.n>0 ) break; + assert( *pRc || pLayer->block.nAlloc==0 ); + assert( *pRc || pLayer->key.nAlloc==0 ); + sqlite3_free(pLayer->block.a); + sqlite3_free(pLayer->key.a); } /* Empty output segment. This is a no-op. */ @@ -3628,22 +3637,20 @@ static int fts3TermCmp( } -static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pRc){ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ int bRes = 0; - if( *pRc==SQLITE_OK ){ - sqlite3_stmt *pCheck = 0; - int rc; + sqlite3_stmt *pCheck = 0; + int rc; - rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCheck, 1, iEnd); - if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; - rc = sqlite3_reset(pCheck); - } - *pRc = rc; + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); } - return bRes; + *pbRes = bRes; + return rc; } /* @@ -3682,16 +3689,14 @@ static int fts3IncrmergeLoad( } /* Check for the zero-length marker in the %_segments table */ - bAppendable = fts3IsAppendable(p, iEnd, &rc); + rc = fts3IsAppendable(p, iEnd, &bAppendable); /* Check that zKey/nKey is larger than the largest key the candidate */ if( rc==SQLITE_OK && bAppendable ){ - char *aLeaf = (char *)aRoot; - int nLeaf = nRoot; + char *aLeaf = 0; + int nLeaf = 0; - if( aRoot[0] ){ - rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); - } + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); if( rc==SQLITE_OK ){ NodeReader reader; for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); @@ -3705,7 +3710,7 @@ static int fts3IncrmergeLoad( } nodeReaderRelease(&reader); } - if( aLeaf!=aRoot ) sqlite3_free(aLeaf); + sqlite3_free(aLeaf); } if( rc==SQLITE_OK && bAppendable ){ @@ -3866,9 +3871,8 @@ static int fts3IncrmergeWriter( /* Insert the marker in the %_segments table to make sure nobody tries ** to steal the space just allocated. This is also used to identify ** appendable segments. */ - if( rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); - } + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; pWriter->iAbsLevel = iAbsLevel; pWriter->nLeafEst = nLeafEst; @@ -4010,6 +4014,7 @@ static int fts3TruncateNode( pNew, &prev, reader.term.a, reader.term.n, reader.aDoclist, reader.nDoclist ); + if( rc!=SQLITE_OK ) break; } if( bStarted==0 ){ fts3StartNode(pNew, (int)aNode[0], reader.iChild); @@ -4029,68 +4034,66 @@ static int fts3TruncateSegment( const char *zTerm, /* Remove terms smaller than this */ int nTerm /* Number of bytes in buffer zTerm */ ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ Blob root = {0,0,0}; /* New root page image */ Blob block = {0,0,0}; /* Buffer used for any other block */ - + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + int rc2; /* sqlite3_reset() return code */ sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); - if( rc==SQLITE_OK ){ - sqlite3_int64 iBlock = 0; /* Block id */ - sqlite3_int64 iNewStart = 0; - sqlite3_int64 iOldStart = 0; - int rc2; /* sqlite3_reset() return code */ + assert( p->aStmt[SQL_SELECT_SEGDIR] ); + pFetch = p->aStmt[SQL_SELECT_SEGDIR]; - sqlite3_bind_int64(pFetch, 1, iAbsLevel); - sqlite3_bind_int(pFetch, 2, iIdx); - if( SQLITE_ROW==sqlite3_step(pFetch) ){ - const char *aRoot = sqlite3_column_blob(pFetch, 4); - int nRoot = sqlite3_column_bytes(pFetch, 4); - iOldStart = sqlite3_column_int64(pFetch, 1); - rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); - } - rc2 = sqlite3_reset(pFetch); - if( rc==SQLITE_OK ) rc = rc2; + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; - while( rc==SQLITE_OK && iBlock ){ - char *aBlock = 0; - int nBlock = 0; - iNewStart = iBlock; - - rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); - if( rc==SQLITE_OK ){ - rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); - } - if( rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, iNewStart, block.a, block.n); - } - sqlite3_free(aBlock); - } - - /* Variable iNewStart now contains the first valid leaf node. */ - if( rc==SQLITE_OK && iNewStart ){ - sqlite3_stmt *pDel = 0; - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDel, 1, iOldStart); - sqlite3_bind_int64(pDel, 2, iNewStart-1); - sqlite3_step(pDel); - rc = sqlite3_reset(pDel); - } - } + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); if( rc==SQLITE_OK ){ - sqlite3_stmt *pChomp = 0; - rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pChomp, 1, iNewStart); - sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); - sqlite3_bind_int64(pChomp, 3, iAbsLevel); - sqlite3_bind_int(pChomp, 4, iIdx); - sqlite3_step(pChomp); - rc = sqlite3_reset(pChomp); - } + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); } } @@ -4122,7 +4125,7 @@ static int fts3IncrmergeChomp( || (pCsr->apSegment[1]->iIdx==0 && pCsr->apSegment[0]->iIdx==1) ); - for(i=1; i>=0; i--){ + for(i=1; i>=0 && rc==SQLITE_OK; i--){ Fts3SegReader *pSeg = pCsr->apSegment[0]; if( pSeg->iIdx!=i ) pSeg = pCsr->apSegment[1]; assert( pSeg->iIdx==i ); @@ -4149,10 +4152,10 @@ static int fts3IncrmergeChomp( /* ** Attempt an incremental merge that writes nMerge leaf pages. ** -** Incremental merges happen two segments at a time. The two +** Incremental merges happen two segments at a time. The two ** segments to be merged are the two oldest segments (the ones with ** the smallest index) in the highest level that has at least -** nMin segments. Multiple segment pair merges might occur in +** nMin segments. Multiple segment pair merges might occur in ** an attempt to write the quota of nMerge leaf pages. */ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ @@ -4178,8 +4181,7 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ return sqlite3_reset(pFindLevel); } iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - rc = sqlite3_reset(pFindLevel); - if( rc!=SQLITE_OK ) return rc; + sqlite3_reset(pFindLevel); /* Allocate space for the cursor, filter and writer objects */ pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); @@ -4195,6 +4197,8 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter); + } + if( rc==SQLITE_OK ){ if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr->zTerm,pCsr->nTerm,pWriter); if( rc==SQLITE_OK ){ @@ -4230,7 +4234,10 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** written for the merge, and the minimum number of segments on a level ** before it will be selected for a merge, respectively. */ -static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ int rc; int nMin = (FTS3_MERGE_COUNT / 2); int nMerge = 0; @@ -4252,11 +4259,12 @@ static int fts3DoIncrmerge(Fts3Table *p, const char *zParam){ } } - if( z[0]!='\0' ) return SQLITE_ERROR; - if( nMin<2 ) nMin = 2; - - rc = fts3Incrmerge(p, nMerge, nMin); - sqlite3Fts3SegmentsClose(p); + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = fts3Incrmerge(p, nMerge, nMin); + sqlite3Fts3SegmentsClose(p); + } return rc; } @@ -4476,7 +4484,7 @@ static int fts3DeleteByRowid( ** tables. The schema of the virtual table being: ** ** CREATE TABLE ( -** , +** , **
HIDDEN, ** docid HIDDEN, ** HIDDEN diff --git a/manifest b/manifest index 2ce91db0a4..06d7ae13ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sallocating\sa\slarge\sobject\son\sthe\sstack\sin\sthe\sincremental\smerge\scode.\sUse\ssqlite3_malloc()\sinstead. -D 2012-03-14T12:17:40.405 +C Add\stests\sfor\sincremental\smerge\scode. +D 2012-03-14T20:01:52.250 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -65,7 +65,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 806632fd0020eed966ab82ea25fe09f1a4c86907 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h cc8991daf660b926ef77a004fe92b97adaf10d97 +F ext/fts3/fts3Int.h 1da6d2af6b079cdd74cd2350761182dedb8bd892 F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 772b8c32b93c60b85b60c635de2ff5b3f49fd779 +F ext/fts3/fts3_write.c 355121666034ece7d2472caa3f322d0cb61b0f0d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -448,7 +448,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 -F test/fts3_common.tcl 91c29230c428443e6552add9b18cf94a3dc23074 +F test/fts3_common.tcl 1b2e522476236a018c1a4b8ae4f7599ce29b7be0 F test/fts3aa.test 909d5f530d30a8e36b9328d67285eae6537c79c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 @@ -497,7 +497,8 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a -F test/fts4merge.test 3af8fa8fd9f27a9eb402b9ae399aa53fbaae8481 +F test/fts4merge.test d841f283bf9557d549afde513fe38d0b48adc4f7 +F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -636,7 +637,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 2b5a1b64a8e5114757457fbce9010387d1fe7682 +F test/permutations.test 0ab1e7748de5d29c4c648ba5ce3b983ab80653d1 F test/pragma.test f11c59ec935a52edb4d3d5676d456588121fcefa F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -993,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f97b12e0955c4c29f9c31a186d72d87f7407782e -R 86bb2bcb41aa5bde569f19117ae756eb +P 36ae510de45be44efd34cff242d02fb21b7419ac +R 326f29ae3378aab5424c0bd2deaa1142 U dan -Z da9e601d088fda8ad57ce1fc3070902a +Z 9fe983241614f7794efd59c0386308c4 diff --git a/manifest.uuid b/manifest.uuid index 137182d9b7..9cec3849f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36ae510de45be44efd34cff242d02fb21b7419ac \ No newline at end of file +570473729d6561d81e6e5f8884fd18487008636e \ No newline at end of file diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index 23a8cfca67..6c9c2fcec9 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -14,6 +14,67 @@ # to use Tcl. # + +#------------------------------------------------------------------------- +# USAGE: fts3_build_db_1 N +# +# Build a sample FTS table in the database opened by database connection +# [db]. The name of the new table is "t1". +# +proc fts3_build_db_1 {n} { + + if {$n > 10000} {error "n must be <= 10000"} + + db eval { CREATE VIRTUAL TABLE t1 USING fts4(x, y) } + + set xwords [list zero one two three four five six seven eight nine ten] + set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] + + for {set i 0} {$i < $n} {incr i} { + set x "" + set y "" + + set x [list] + lappend x [lindex $xwords [expr ($i / 1000) % 10]] + lappend x [lindex $xwords [expr ($i / 100) % 10]] + lappend x [lindex $xwords [expr ($i / 10) % 10]] + lappend x [lindex $xwords [expr ($i / 1) % 10]] + + set y [list] + lappend y [lindex $ywords [expr ($i / 1000) % 10]] + lappend y [lindex $ywords [expr ($i / 100) % 10]] + lappend y [lindex $ywords [expr ($i / 10) % 10]] + lappend y [lindex $ywords [expr ($i / 1) % 10]] + + db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) } + } +} + +#------------------------------------------------------------------------- +# USAGE: fts3_build_db_2 N +# +# Build a sample FTS table in the database opened by database connection +# [db]. The name of the new table is "t2". +# +proc fts3_build_db_2 {n} { + + if {$n > 100000} {error "n must be <= 100000"} + + db eval { CREATE VIRTUAL TABLE t2 USING fts4 } + + set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z ""] + + for {set i 0} {$i < $n} {incr i} { + set word "" + set n [llength $chars] + append word [lindex $chars [expr {($i / 1) % $n}]] + append word [lindex $chars [expr {($i / $n) % $n}]] + append word [lindex $chars [expr {($i / ($n*$n)) % $n}]] + + db eval { INSERT INTO t2(docid, content) VALUES($i, $word) } + } +} + #------------------------------------------------------------------------- # USAGE: fts3_integrity_check TBL # @@ -98,7 +159,7 @@ proc fts3_integrity_check {tbl} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing" lappend errors $es } else { - if {$C($iDoc,$iCol,$pos) != "$term"} { + if {[string compare $C($iDoc,$iCol,$pos) $term]} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index " append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\"" lappend errors $es diff --git a/test/fts4merge.test b/test/fts4merge.test index 805a78cf97..a5995921f3 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -23,35 +23,6 @@ ifcapable !fts3 { return } -proc fts3_build_db_1 {n} { - - if {$n > 10000} {error "n must be <= 10000"} - - db eval { CREATE VIRTUAL TABLE t1 USING fts4(x, y) } - - set xwords [list zero one two three four five six seven eight nine ten] - set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] - - for {set i 0} {$i < $n} {incr i} { - set x "" - set y "" - - set x [list] - lappend x [lindex $xwords [expr ($i / 1000) % 10]] - lappend x [lindex $xwords [expr ($i / 100) % 10]] - lappend x [lindex $xwords [expr ($i / 10) % 10]] - lappend x [lindex $xwords [expr ($i / 1) % 10]] - - set y [list] - lappend y [lindex $ywords [expr ($i / 1000) % 10]] - lappend y [lindex $ywords [expr ($i / 100) % 10]] - lappend y [lindex $ywords [expr ($i / 10) % 10]] - lappend y [lindex $ywords [expr ($i / 1) % 10]] - - db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) } - } -} - #------------------------------------------------------------------------- # Test cases 1.* # @@ -98,6 +69,57 @@ do_execsql_test 1.5 { 3 {0 1 2} } +#------------------------------------------------------------------------- +# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are +# handled correctly. +# +do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts4 } + +foreach {tn arg} { + 1 {merge=abc} + 2 {merge=%%%} + 3 {merge=,} + 4 {merge=5,} + 5 {merge=6,%} + 6 {merge=6,six} + 7 {merge=6,1} + 8 {merge=6,0} +} { + do_catchsql_test 2.$tn { + INSERT INTO t2(t2) VALUES($arg); + } {1 {SQL logic error or missing database}} +} + +#------------------------------------------------------------------------- +# Test cases 3.* +# +do_test 3.0 { + reset_db + execsql { PRAGMA page_size = 512 } + fts3_build_db_2 30040 +} {} +do_test 3.1 { fts3_integrity_check t2 } {ok} + +do_execsql_test 3.2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4} + 2 {0 1 2 3 4} + 3 {0 1 2 3 4 5 6} +} + +do_execsql_test 3.3 { + INSERT INTO t2(t2) VALUES('merge=1000000,2'); + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level +} { + 0 {0 1} + 1 {0 1} + 2 0 + 3 {0 1} + 4 {0 1} + 5 0 +} finish_test diff --git a/test/fts4merge2.test b/test/fts4merge2.test new file mode 100644 index 0000000000..308b6929fa --- /dev/null +++ b/test/fts4merge2.test @@ -0,0 +1,38 @@ + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl +set ::testprefix fts4merge2 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_test 1.0 { + fts3_build_db_1 1000 + faultsim_save_and_close +} {} + +do_faultsim_test 1.1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t1(t1) VALUES('merge=32,4') } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 1.2 -faults oom-t* -prep { + if {$iFail<100} {set iFail 803} + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t1(t1) VALUES('merge=1,2') } + execsql { INSERT INTO t1(t1) VALUES('merge=1,2') } +} -test { + faultsim_test_result {0 {}} +} + +finish_test diff --git a/test/permutations.test b/test/permutations.test index 26c1b2a514..6f66e85e94 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -184,7 +184,7 @@ test_suite "fts3" -prefix "" -description { fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test - fts3corrupt2.test fts3first.test fts4langid.test + fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test } From 4ab6f2b9c1fba9d2a40d7b15c4c8b1742bf16dcb Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Mar 2012 17:45:50 +0000 Subject: [PATCH 08/68] Modify incremental merge code to merge nMin segments at a time. FossilOrigin-Name: cd34bc1af4ba608ea3b52bab55bcfe0086711900 --- ext/fts3/fts3_write.c | 58 +++++++++++++++++++++++++++---------------- manifest | 16 ++++++------ manifest.uuid | 2 +- test/fts3_common.tcl | 8 +++--- test/fts4merge.test | 18 +++++++------- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 08f07c10b2..a6ee1f6919 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -306,10 +306,10 @@ static int fts3SqlStmt( " ORDER BY (level %% 1024) DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment -** created by merging the two segments with idx=0 and idx=1 from absolute -** level ?. See function fts3Incrmerge() for details. */ +** created by merging the oldest :2 segments from absolute level :1. See +** function fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " - " FROM %Q.'%q_segdir' WHERE level = ? AND idx BETWEEN 0 AND 1", + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", /* SQL_DELETE_SEGDIR_ENTRY ** Delete the %_segdir entry on absolute level :1 with index :2. */ @@ -1077,6 +1077,9 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ +sqlite3_log(SQLITE_OK, + "16-way merge at level=%d langid=%d index=%d", iLevel, iLangid, iIndex +); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ @@ -3232,25 +3235,30 @@ static int fts3DoRebuild(Fts3Table *p){ static int fts3IncrmergeCsr( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc; /* Return Code */ - sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + assert( nSeg>=2 ); memset(pCsr, 0, sizeof(*pCsr)); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader *)*2); + nByte = sizeof(Fts3SegReader *) * nSeg; + + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ - memset(pCsr->apSegment, 0, sizeof(Fts3SegReader *)*2); - pCsr->nSegment = 2; + memset(pCsr->apSegment, 0, nByte); + pCsr->nSegment = nSeg; rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); } if( rc==SQLITE_OK ){ int i; int rc2; sqlite3_bind_int64(pStmt, 1, iAbsLevel); - for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<2; i++){ + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && izTerm; + int nKey = pCsr->nTerm; rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ @@ -3849,6 +3858,7 @@ static int fts3IncrmergeWriter( rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ nLeafEst = sqlite3_column_int(pLeafEst, 0); } @@ -4120,15 +4130,17 @@ static int fts3IncrmergeChomp( int i; int rc = SQLITE_OK; - assert( pCsr->nSegment==2 ); - assert( (pCsr->apSegment[0]->iIdx==0 && pCsr->apSegment[1]->iIdx==1) - || (pCsr->apSegment[1]->iIdx==0 && pCsr->apSegment[0]->iIdx==1) - ); + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; - for(i=1; i>=0 && rc==SQLITE_OK; i--){ - Fts3SegReader *pSeg = pCsr->apSegment[0]; - if( pSeg->iIdx!=i ) pSeg = pCsr->apSegment[1]; - assert( pSeg->iIdx==i ); + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); if( pSeg->aNode==0 ){ /* Seg-reader is at EOF. Remove the entire input segment. */ @@ -4178,6 +4190,8 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); sqlite3_bind_int(pFindLevel, 1, nMin); if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ + /* There are no levels with nMin or more segments. Or an error has + ** occurred. Either way, exit early. */ return sqlite3_reset(pFindLevel); } iAbsLevel = sqlite3_column_int64(pFindLevel, 0); @@ -4193,14 +4207,16 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Open a cursor to iterate through the contents of indexes 0 and 1 of ** the selected absolute level. */ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; - rc = fts3IncrmergeCsr(p, iAbsLevel, pCsr); - + rc = fts3IncrmergeCsr(p, iAbsLevel, nMin, pCsr); +sqlite3_log(SQLITE_OK, "%d-way merge from level=%d to level=%d", + nMin, (int)iAbsLevel, (int)iAbsLevel+1 +); if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter); } if( rc==SQLITE_OK ){ if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr->zTerm,pCsr->nTerm,pWriter); + rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr, pWriter); if( rc==SQLITE_OK ){ do { rc = fts3IncrmergeAppend(p, pWriter, pCsr); diff --git a/manifest b/manifest index 06d7ae13ce..88b357ba30 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sincremental\smerge\scode. -D 2012-03-14T20:01:52.250 +C Modify\sincremental\smerge\scode\sto\smerge\snMin\ssegments\sat\sa\stime. +D 2012-03-15T17:45:50.857 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 355121666034ece7d2472caa3f322d0cb61b0f0d +F ext/fts3/fts3_write.c be60aaf4a887a68aaf0d7541bc78413e7ba66291 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -448,7 +448,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 -F test/fts3_common.tcl 1b2e522476236a018c1a4b8ae4f7599ce29b7be0 +F test/fts3_common.tcl d65de7e21c2b933bf17152830eb924356f197c8f F test/fts3aa.test 909d5f530d30a8e36b9328d67285eae6537c79c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 @@ -497,7 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a -F test/fts4merge.test d841f283bf9557d549afde513fe38d0b48adc4f7 +F test/fts4merge.test 1c3389a3be18498934baf76afe67663d8b4b5e42 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -994,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 36ae510de45be44efd34cff242d02fb21b7419ac -R 326f29ae3378aab5424c0bd2deaa1142 +P 570473729d6561d81e6e5f8884fd18487008636e +R 8deb5f38fb61fdd986c036580c00c7d8 U dan -Z 9fe983241614f7794efd59c0386308c4 +Z c18f3e75d81727125e658ae77334fefb diff --git a/manifest.uuid b/manifest.uuid index 9cec3849f5..08619b20e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -570473729d6561d81e6e5f8884fd18487008636e \ No newline at end of file +cd34bc1af4ba608ea3b52bab55bcfe0086711900 \ No newline at end of file diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index 6c9c2fcec9..f031f927d6 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -66,10 +66,10 @@ proc fts3_build_db_2 {n} { for {set i 0} {$i < $n} {incr i} { set word "" - set n [llength $chars] - append word [lindex $chars [expr {($i / 1) % $n}]] - append word [lindex $chars [expr {($i / $n) % $n}]] - append word [lindex $chars [expr {($i / ($n*$n)) % $n}]] + set nChar [llength $chars] + append word [lindex $chars [expr {($i / 1) % $nChar}]] + append word [lindex $chars [expr {($i / $nChar) % $nChar}]] + append word [lindex $chars [expr {($i / ($nChar*$nChar)) % $nChar}]] db eval { INSERT INTO t2(docid, content) VALUES($i, $word) } } diff --git a/test/fts4merge.test b/test/fts4merge.test index a5995921f3..9f7e05b0ed 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -47,9 +47,9 @@ for {set i 0} {$i<20} {incr i} { do_execsql_test 1.3 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level } { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7} - 2 {0 1 2 3 4 5 6} + 0 {0 1 2 3} + 1 {0 1 2 3 4 5 6} + 2 {0 1 2 3} } for {set i 0} {$i<100} {incr i} { @@ -64,9 +64,9 @@ do_execsql_test 1.5 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level } { 0 {0 1 2 3} - 1 {0 1 2 3} - 2 {0 1 2 3} - 3 {0 1 2} + 1 {0 1 2} + 2 0 + 3 0 } #------------------------------------------------------------------------- @@ -103,7 +103,7 @@ do_test 3.1 { fts3_integrity_check t2 } {ok} do_execsql_test 3.2 { SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level } { - 0 {0 1 2 3 4 5 6 7} + 0 {0 1 2 3 4 5 6} 1 {0 1 2 3 4} 2 {0 1 2 3 4} 3 {0 1 2 3 4 5 6} @@ -113,10 +113,10 @@ do_execsql_test 3.3 { INSERT INTO t2(t2) VALUES('merge=1000000,2'); SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level } { - 0 {0 1} + 0 0 1 {0 1} 2 0 - 3 {0 1} + 3 {0 1} 4 {0 1} 5 0 } From e931b7f45fd2560baf2c47c1d1546ce14dddf21b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Mar 2012 14:54:07 +0000 Subject: [PATCH 09/68] Add a comment to the FTS getAbsoluteLevel() function. No actual code changes. FossilOrigin-Name: 7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd --- ext/fts3/fts3_write.c | 31 +++++++++++++++++++++++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index a6ee1f6919..f9cb3667c6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -481,11 +481,34 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ return rc; } +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ static sqlite3_int64 getAbsoluteLevel( - Fts3Table *p, - int iLangid, - int iIndex, - int iLevel + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ ){ assert( iLangid>=0 ); assert( p->nIndex>0 ); diff --git a/manifest b/manifest index 88b357ba30..be0bd8c83f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sincremental\smerge\scode\sto\smerge\snMin\ssegments\sat\sa\stime. -D 2012-03-15T17:45:50.857 +C Add\sa\scomment\sto\sthe\sFTS\sgetAbsoluteLevel()\sfunction.\sNo\sactual\scode\schanges. +D 2012-03-16T14:54:07.123 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c be60aaf4a887a68aaf0d7541bc78413e7ba66291 +F ext/fts3/fts3_write.c fab92bcc6de43fb6dfd3937ac67e2cb45c834a2b F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -994,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 570473729d6561d81e6e5f8884fd18487008636e -R 8deb5f38fb61fdd986c036580c00c7d8 +P cd34bc1af4ba608ea3b52bab55bcfe0086711900 +R c9f36e032863c0b8f751fc4c1ff8b5a5 U dan -Z c18f3e75d81727125e658ae77334fefb +Z b14699878441d211e25f75f7c2630c95 diff --git a/manifest.uuid b/manifest.uuid index 08619b20e5..ca4d0738ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd34bc1af4ba608ea3b52bab55bcfe0086711900 \ No newline at end of file +7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd \ No newline at end of file From 81b35dc66c51c9aeb963d90a2216692e0859982d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Mar 2012 15:54:19 +0000 Subject: [PATCH 10/68] Fix some integer overflow problems that can occur when using large langauge id values. FossilOrigin-Name: 3475092cff862080a020d386076d739f0d22c9b2 --- ext/fts3/fts3_write.c | 37 ++++++++------- manifest | 14 +++--- manifest.uuid | 2 +- test/fts4langid.test | 102 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 24 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f9cb3667c6..fca5d9f70e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -510,10 +510,13 @@ static sqlite3_int64 getAbsoluteLevel( int iIndex, /* Index in p->aIndex[] */ int iLevel /* Level of segments */ ){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndexnIndex ); - return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; } /* @@ -556,7 +559,7 @@ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select */ + int iLevel, /* Level to select (relative level) */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; @@ -571,7 +574,7 @@ int sqlite3Fts3AllSegdirs( rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } @@ -579,7 +582,7 @@ int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } *ppStmt = pStmt; @@ -1854,7 +1857,7 @@ static int fts3WriteSegment( */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ - int iLevel, /* Value for "level" field */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ @@ -1865,7 +1868,7 @@ static int fts3WriteSegdir( sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); @@ -2248,7 +2251,7 @@ static int fts3SegWriterAdd( static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - int iLevel, /* Value for 'level' column of %_segdir */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ int iIdx /* Value for 'idx' column of %_segdir */ ){ int rc; /* Return code */ @@ -2330,7 +2333,7 @@ static int fts3SegmentMaxLevel( Fts3Table *p, int iLangid, int iIndex, - int *pnMax + sqlite3_int64 *pnMax ){ sqlite3_stmt *pStmt; int rc; @@ -2344,12 +2347,12 @@ static int fts3SegmentMaxLevel( */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -2414,15 +2417,17 @@ static int fts3DeleteSegdir( if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pDelete, 2, + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } @@ -2899,7 +2904,7 @@ static int fts3SegmentMerge( ){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level/index to create new segment at */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ @@ -3864,7 +3869,7 @@ static int fts3IncrmergeWriter( rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pOutputIdx, 1, iAbsLevel+1); + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); sqlite3_step(pOutputIdx); iIdx = sqlite3_column_int(pOutputIdx, 0) - 1; rc = sqlite3_reset(pOutputIdx); diff --git a/manifest b/manifest index be0bd8c83f..8bb46fe386 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sthe\sFTS\sgetAbsoluteLevel()\sfunction.\sNo\sactual\scode\schanges. -D 2012-03-16T14:54:07.123 +C Fix\ssome\sinteger\soverflow\sproblems\sthat\scan\soccur\swhen\susing\slarge\slangauge\sid\svalues. +D 2012-03-16T15:54:19.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c fab92bcc6de43fb6dfd3937ac67e2cb45c834a2b +F ext/fts3/fts3_write.c 548d034e16ecad0ec5dfd542c48c591564c343d3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a +F test/fts4langid.test 2f6ae2e5ab037b11b411c85346a09a5fee503f6d F test/fts4merge.test 1c3389a3be18498934baf76afe67663d8b4b5e42 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -994,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cd34bc1af4ba608ea3b52bab55bcfe0086711900 -R c9f36e032863c0b8f751fc4c1ff8b5a5 +P 7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd +R 3c4572a4f07a3af6855856a4d0059330 U dan -Z b14699878441d211e25f75f7c2630c95 +Z 5aeec06647e7b858688ed8df27a75aed diff --git a/manifest.uuid b/manifest.uuid index ca4d0738ba..b07a988ed8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd \ No newline at end of file +3475092cff862080a020d386076d739f0d22c9b2 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 08f1a21c17..012924c70d 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -382,4 +382,106 @@ do_catchsql_test 4.1.5 { INSERT INTO t4(content, lid) VALUES('hello world', 101) } {1 {SQL logic error or missing database}} +#------------------------------------------------------------------------- +# Test cases 5.* +# +# The following test cases are designed to detect a 32-bit overflow bug +# that existed at one point. +# +proc build_multilingual_db_3 {db} { + $db eval { + CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid); + } + set languages [list 0 1 2 [expr 1<<30]] + + foreach lid $languages { + execsql { + INSERT INTO t5(docid, content, lid) VALUES( + $lid, 'My language is ' || $lid, $lid + ) + } + } +} + +do_test 5.1.0 { + reset_db + build_multilingual_db_3 db +} {} + +do_execsql_test 5.1.1 { + SELECT level FROM t5_segdir; +} [list 0 1024 2048 [expr 1<<40]] + +do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0 +foreach langid [list 0 1 2 [expr 1<<30]] { + do_execsql_test 5.2.$langid { + SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid + } $langid +} + +set lid [expr 1<<30] +do_execsql_test 5.3.1 { + CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid); + INSERT INTO t6 VALUES('I belong to language 0!'); +} +do_test 5.3.2 { + for {set i 0} {$i < 20} {incr i} { + execsql { + INSERT INTO t6(content, lid) VALUES( + 'I (row '||$i||') belong to langauge N!', $lid + ); + } + } + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } +} {1} + +do_test 5.3.3 { + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid} +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + +do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {} +do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1} +do_execsql_test 5.3.6 { + SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + + +set lid [expr 1<<30] +foreach lid [list 4 [expr 1<<30]] { + do_execsql_test 5.4.$lid.1 { + DELETE FROM t6; + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {0 0} + do_execsql_test 5.4.$lid.2 { + INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero zero one', $lid); + INSERT INTO t6(content, lid) VALUES('zero one zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero one one', $lid); + INSERT INTO t6(content, lid) VALUES('one zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('one zero one', $lid); + INSERT INTO t6(content, lid) VALUES('one one zero', $lid); + INSERT INTO t6(content, lid) VALUES('one one one', $lid); + + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.3 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {8 0} + + do_execsql_test 5.4.$lid.4 { + INSERT INTO t6(t6) VALUES('merge=100,3'); + INSERT INTO t6(t6) VALUES('merge=100,3'); + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.5 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {4 4} +} + + finish_test From d1ab097d458196d9201e3af2f57dfecb73e7df83 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Mar 2012 16:56:57 +0000 Subject: [PATCH 11/68] Fix various incorrect and missing comments and other style issues in and around the FTS incremental merge code. FossilOrigin-Name: 7aabb62c8ccbd2b8d216e25226f06e5820dec38a --- ext/fts3/fts3Int.h | 17 +- ext/fts3/fts3_write.c | 548 +++++++++++++++++++++++++----------------- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 351 insertions(+), 230 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1cd14d304a..e847aad15d 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -213,19 +213,22 @@ struct Fts3Table { char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* TODO: Fix the first paragraph of this comment. - ** + /* ** The following array of hash tables is used to buffer pending index - ** updates during transactions. Variable nPendingData estimates the memory - ** size of the pending data, including hash table overhead, not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, the buffer - ** is flushed automatically. Variable iPrevDocid is the docid of the most - ** recently inserted record. + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. */ int nIndex; /* Size of aIndex[] */ struct Fts3Index { diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index fca5d9f70e..1218814b4a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -66,6 +66,22 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -373,6 +389,7 @@ static int fts3SqlStmt( return rc; } + static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ @@ -519,25 +536,6 @@ static sqlite3_int64 getAbsoluteLevel( return iBase + iLevel; } -/* -** Given an absolute level number, determine the langauge-id, index -** and relative level that it corresponds to. -** -** The return value is the relative level. The language-id and index -** are returned via output variables. -*/ -static int getRelativeLevel( - Fts3Table *p, /* FTS table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level */ - int *piLangid, /* OUT: Language id */ - int *piIndex /* OUT: Index in p->aIndex[] */ -){ - if( piLangid ) *piLangid = (iAbsLevel / FTS3_SEGDIR_MAXLEVEL) / p->nIndex; - if( piIndex ) *piIndex = (iAbsLevel / FTS3_SEGDIR_MAXLEVEL) % p->nIndex; - return iAbsLevel % FTS3_SEGDIR_MAXLEVEL; -} - - /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -1103,9 +1101,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ -sqlite3_log(SQLITE_OK, - "16-way merge at level=%d langid=%d index=%d", iLevel, iLangid, iIndex -); + fts3LogMerge(16, getAbsoluteLevel(iLevel, iLangid, iIndex)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ @@ -3258,7 +3254,8 @@ static int fts3DoRebuild(Fts3Table *p){ /* ** This function opens a cursor used to read the input data for an ** incremental merge operation. Specifically, it opens a cursor to scan -** the oldest two segments (idx=0 and idx=1) in absolute level iAbsLevel. +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. */ static int fts3IncrmergeCsr( Fts3Table *p, /* FTS3 table handle */ @@ -3270,11 +3267,12 @@ static int fts3IncrmergeCsr( sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ int nByte; /* Bytes allocated at pCsr->apSegment[] */ + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ assert( nSeg>=2 ); memset(pCsr, 0, sizeof(*pCsr)); nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ @@ -3304,7 +3302,7 @@ static int fts3IncrmergeCsr( } typedef struct IncrmergeWriter IncrmergeWriter; -typedef struct LayerWriter LayerWriter; +typedef struct NodeWriter NodeWriter; typedef struct Blob Blob; typedef struct NodeReader NodeReader; @@ -3320,12 +3318,20 @@ struct Blob { int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ }; -struct LayerWriter { +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { sqlite3_int64 iBlock; /* Current block id */ Blob key; /* Last key written to the current block */ Blob block; /* Current block image */ }; +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ struct IncrmergeWriter { int nLeafEst; /* Space allocated for leaf blocks */ int nWork; /* Number of leaf pages flushed */ @@ -3333,7 +3339,7 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ - LayerWriter aLayer[FTS_MAX_APPENDABLE_HEIGHT]; + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* @@ -3356,6 +3362,15 @@ struct NodeReader { int nDoclist; /* Size of doclist in bytes */ }; +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; @@ -3369,6 +3384,16 @@ static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ } } +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ static int nodeReaderNext(NodeReader *p){ int bFirst = (p->term.n==0); /* True for first term on the node */ int nPrefix = 0; /* Bytes to copy from previous term */ @@ -3404,12 +3429,19 @@ static int nodeReaderNext(NodeReader *p){ return rc; } +/* +** Release all dynamic resources held by node-reader object *p. +*/ static void nodeReaderRelease(NodeReader *p){ sqlite3_free(p->term.a); } /* -** Initialize a node-reader object. +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. */ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ memset(p, 0, sizeof(NodeReader)); @@ -3435,7 +3467,7 @@ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ ** node. ** ** The block id of the leaf node just written to disk may be found in -** (pWriter->aLayer[0].iBlock) when this function is called. +** (pWriter->aNodeWriter[0].iBlock) when this function is called. */ static int fts3IncrmergePush( Fts3Table *p, /* Fts3 table handle */ @@ -3443,13 +3475,13 @@ static int fts3IncrmergePush( const char *zTerm, /* Term to write to internal node */ int nTerm /* Bytes at zTerm */ ){ - sqlite3_int64 iPtr = pWriter->aLayer[0].iBlock; + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; int iLayer; assert( nTerm>0 ); for(iLayer=1; ALWAYS(iLayeraLayer[iLayer]; + NodeWriter *pNode = &pWriter->aNodeWriter[iLayer]; int rc = SQLITE_OK; int nPrefix; int nSuffix; @@ -3459,17 +3491,17 @@ static int fts3IncrmergePush( ** the current node of layer iLayer. Due to the prefix compression, ** the space required changes depending on which node the key is to ** be added to. */ - nPrefix = fts3PrefixCompress(pLayer->key.a, pLayer->key.n, zTerm, nTerm); + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); nSuffix = nTerm - nPrefix; nSpace = sqlite3Fts3VarintLen(nPrefix); nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - if( pLayer->key.n==0 || (pLayer->block.n + nSpace)<=p->nNodeSize ){ + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ /* If the current node of layer iLayer contains zero keys, or if adding ** the key to it will not cause it to grow to larger than nNodeSize ** bytes in size, write the key here. */ - Blob *pBlk = &pLayer->block; + Blob *pBlk = &pNode->block; if( pBlk->n==0 ){ blobGrowBuffer(pBlk, p->nNodeSize, &rc); if( rc==SQLITE_OK ){ @@ -3478,32 +3510,32 @@ static int fts3IncrmergePush( } } blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); - blobGrowBuffer(&pLayer->key, nTerm, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); if( rc==SQLITE_OK ){ - if( pLayer->key.n ){ + if( pNode->key.n ){ pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); } pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); pBlk->n += nSuffix; - memcpy(pLayer->key.a, zTerm, nTerm); - pLayer->key.n = nTerm; + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; } }else{ /* Otherwise, flush the the current node of layer iLayer to disk. ** Then allocate a new, empty sibling node. The key will be written ** into the parent of this node. */ - rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a,pLayer->block.n); + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); - assert( pLayer->block.nAlloc>=p->nNodeSize ); - pLayer->block.a[0] = (char)iLayer; - pLayer->block.n = 1 + sqlite3Fts3PutVarint(&pLayer->block.a[1], iPtr+1); + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); - iNextPtr = pLayer->iBlock; - pLayer->iBlock++; - pLayer->key.n = 0; + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; } if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; @@ -3513,6 +3545,80 @@ static int fts3IncrmergePush( assert( 0 ); } +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ static int fts3IncrmergeAppend( Fts3Table *p, /* Fts3 table handle */ IncrmergeWriter *pWriter, /* Writer object */ @@ -3522,16 +3628,14 @@ static int fts3IncrmergeAppend( int nTerm = pCsr->nTerm; const char *aDoclist = pCsr->aDoclist; int nDoclist = pCsr->nDoclist; - int rc = SQLITE_OK; /* Return code */ int nSpace; /* Total space in bytes required on leaf */ - int nPrefix; - int nSuffix; - LayerWriter *pLayer; - Blob *pPg; + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ - pLayer = &pWriter->aLayer[0]; - nPrefix = fts3PrefixCompress(pLayer->key.a, pLayer->key.n, zTerm, nTerm); + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); nSuffix = nTerm - nPrefix; nSpace = sqlite3Fts3VarintLen(nPrefix); @@ -3541,15 +3645,15 @@ static int fts3IncrmergeAppend( /* If the current block is not empty, and if adding this term/doclist ** to the current block would make it larger than Fts3Table.nNodeSize ** bytes, write this block out to the database. */ - if( pLayer->block.n>0 && (pLayer->block.n + nSpace)>p->nNodeSize ){ - rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a, pLayer->block.n); + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); pWriter->nWork++; /* Add the current term to the parent node. The term added to the ** parent must: ** ** a) be greater than the largest term on the leaf node just written - ** to the database (still available in pLayer->key), and + ** to the database (still available in pLeaf->key), and ** ** b) be less than or equal to the term about to be added to the new ** leaf node (zTerm/nTerm). @@ -3562,9 +3666,9 @@ static int fts3IncrmergeAppend( } /* Advance to the next output block */ - pLayer->iBlock++; - pLayer->key.n = 0; - pLayer->block.n = 0; + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; nPrefix = 0; nSuffix = nTerm; @@ -3573,72 +3677,96 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } - blobGrowBuffer(&pLayer->key, nTerm, &rc); - blobGrowBuffer(&pLayer->block, pLayer->block.n + nSpace, &rc); + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); if( rc==SQLITE_OK ){ - /* Update the block image with the new entry */ - pPg = &pLayer->block; - pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nPrefix); - pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nSuffix); - memcpy(&pPg->a[pPg->n], &zTerm[nPrefix], nSuffix); - pPg->n += nSuffix; - pPg->n += sqlite3Fts3PutVarint(&pPg->a[pPg->n], nDoclist); - memcpy(&pPg->a[pPg->n], aDoclist, nDoclist); - pPg->n += nDoclist; - - /* Take a copy of the key just written */ - memcpy(pLayer->key.a, zTerm, nTerm); - pLayer->key.n = nTerm; + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); } return rc; } +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ static void fts3IncrmergeRelease( - Fts3Table *p, - IncrmergeWriter *pWriter, - int *pRc + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ ){ int i; /* Used to iterate through non-root layers */ - int iRoot; - LayerWriter *pRoot; - int rc = *pRc; + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ - /* Find the root node */ + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ - LayerWriter *pLayer = &pWriter->aLayer[iRoot]; - if( pLayer->block.n>0 ) break; - assert( *pRc || pLayer->block.nAlloc==0 ); - assert( *pRc || pLayer->key.nAlloc==0 ); - sqlite3_free(pLayer->block.a); - sqlite3_free(pLayer->key.a); + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } /* Empty output segment. This is a no-op. */ if( iRoot<0 ) return; - /* The entire output segment fits on the root node. This is not allowed. */ + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ if( iRoot==0 ){ - Blob *pBlock = &pWriter->aLayer[1].block; + Blob *pBlock = &pWriter->aNodeWriter[1].block; blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); if( rc==SQLITE_OK ){ pBlock->a[0] = 0x01; pBlock->n = 1 + sqlite3Fts3PutVarint( - &pBlock->a[1], pWriter->aLayer[0].iBlock + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock ); } iRoot = 1; } - pRoot = &pWriter->aLayer[iRoot]; + pRoot = &pWriter->aNodeWriter[iRoot]; + /* Flush all currently outstanding nodes to disk. */ for(i=0; iaLayer[i]; - if( pLayer->block.n>0 && rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, pLayer->iBlock, pLayer->block.a,pLayer->block.n); + NodeWriter *pNode = &pWriter->aNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); } - sqlite3_free(pLayer->block.a); - sqlite3_free(pLayer->key.a); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } /* Write the %_segdir record. */ @@ -3647,7 +3775,7 @@ static void fts3IncrmergeRelease( pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ - pWriter->aLayer[0].iBlock, /* leaves_end_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); @@ -3658,7 +3786,14 @@ static void fts3IncrmergeRelease( *pRc = rc; } - +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ static int fts3TermCmp( const char *zLhs, int nLhs, /* LHS of comparison */ const char *zRhs, int nRhs /* RHS of comparison */ @@ -3673,10 +3808,22 @@ static int fts3TermCmp( } +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ - int bRes = 0; - sqlite3_stmt *pCheck = 0; - int rc; + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); if( rc==SQLITE_OK ){ @@ -3690,7 +3837,19 @@ static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ } /* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. ** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. */ static int fts3IncrmergeLoad( Fts3Table *p, /* Fts3 table handle */ @@ -3700,18 +3859,20 @@ static int fts3IncrmergeLoad( int nKey, /* Number of bytes in nKey */ IncrmergeWriter *pWriter /* Populate this object */ ){ - sqlite3_int64 iStart = 0; - sqlite3_int64 iLeafEnd = 0; - sqlite3_int64 iEnd = 0; - const char *aRoot = 0; - int nRoot = 0; - int rc; + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ - sqlite3_stmt *pSelect = 0; rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); if( rc==SQLITE_OK ){ - int rc2; - int bAppendable = 0; + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ @@ -3754,7 +3915,7 @@ static int fts3IncrmergeLoad( ** object to do so. */ int i; int nHeight = (int)aRoot[0]; - LayerWriter *pLayer; + NodeWriter *pNode; pWriter->nLeafEst = ((iEnd - iStart) + 1) / FTS_MAX_APPENDABLE_HEIGHT; pWriter->iStart = iStart; @@ -3763,37 +3924,37 @@ static int fts3IncrmergeLoad( pWriter->iIdx = iIdx; for(i=nHeight+1; iaLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } - pLayer = &pWriter->aLayer[nHeight]; - pLayer->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; - blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc); + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); if( rc==SQLITE_OK ){ - memcpy(pLayer->block.a, aRoot, nRoot); - pLayer->block.n = nRoot; + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; } for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ - pLayer = &pWriter->aLayer[i]; + pNode = &pWriter->aNodeWriter[i]; NodeReader reader; - rc = nodeReaderInit(&reader, pLayer->block.a, pLayer->block.n); + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); - blobGrowBuffer(&pLayer->key, reader.term.n, &rc); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); if( rc==SQLITE_OK ){ - memcpy(pLayer->key.a, reader.term.a, reader.term.n); - pLayer->key.n = reader.term.n; + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; if( i>0 ){ char *aBlock = 0; int nBlock = 0; - pLayer = &pWriter->aLayer[i-1]; - pLayer->iBlock = reader.iChild; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pLayer->block, MAX(nBlock, p->nNodeSize), &rc); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); if( rc==SQLITE_OK ){ - memcpy(pLayer->block.a, aBlock, nBlock); - pLayer->block.n = nBlock; + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; } sqlite3_free(aBlock); } @@ -3811,22 +3972,9 @@ static int fts3IncrmergeLoad( /* ** Either allocate an output segment or locate an existing appendable -** output segment to append to. And "appendable" output segment is +** output segment to append to. An "appendable" output segment is ** slightly different to a normal one, as the required range of keys in -** the %_segments table must be allocated up front. This requires some -** assumptions: -** -** * It is expected that due to the short-keys used, and the prefix and -** suffix compression, the fanout of segment b-trees will be very high. -** With a conservative assumption of 32 bytes per key and 1024 byte -** pages, say 32 (2^5). Since SQLite database files are limited to -** a total of 2^31 pages in size, it seems very likely that no segment -** b-tree will have more than ten layers of nodes (including the -** leaves). -** -** * Since each interior node has a pointer to at least two child nodes, -** each layer of interior nodes must be smaller than the layer of -** leaf nodes. +** the %_segments table must be allocated up front. ** ** In the %_segdir table, a segment is defined by the values in three ** columns: @@ -3837,18 +3985,17 @@ static int fts3IncrmergeLoad( ** ** When an appendable segment is allocated, it is estimated that the ** maximum number of leaf blocks that may be required is the sum of the -** number of leaf blocks consumed by the two input segments multiplied -** by three. If an input segment consists of a root node only, treat it -** as if it has a single leaf node for the purposes of this estimate. -** This value is stored in stack variable nLeafEst. +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. ** -** A total of 10*nLeafEst blocks are allocated when an appendable segment -** is created ((1 + end_block - start_block)==10*nLeafEst). The contiguous +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous ** array of leaf nodes starts at the first block allocated. The array ** of interior nodes that are parents of the leaf nodes start at block -** (start_block + (1 + end_block - start_block) / 10). And so on. +** (start_block + (1 + end_block - start_block) / 16). And so on. ** -** In the actual code below, the value "10" is replaced with the +** In the actual code below, the value "16" is replaced with the ** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ static int fts3IncrmergeWriter( @@ -3864,8 +4011,8 @@ static int fts3IncrmergeWriter( sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; + const char *zKey = pCsr->zTerm; /* First key to be appended to output */ + int nKey = pCsr->nTerm; /* Size of zKey in bytes */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ @@ -3916,9 +4063,9 @@ static int fts3IncrmergeWriter( pWriter->nLeafEst = nLeafEst; pWriter->iIdx = iIdx; - /* Set up the array of LayerWriter objects */ + /* Set up the array of NodeWriter objects */ for(i=0; iaLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } return SQLITE_OK; } @@ -3929,15 +4076,19 @@ static int fts3IncrmergeWriter( ** ** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx ** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. */ static int fts3RemoveSegdirEntry( - Fts3Table *p, - sqlite3_int64 iAbsLevel, - int iIdx + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ ){ - int rc; - sqlite3_stmt *pDelete = 0; - sqlite3_stmt *pUpdate = 0; + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + sqlite3_stmt *pUpdate = 0; /* UPDATE statement */ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); if( rc==SQLITE_OK ){ @@ -3960,43 +4111,6 @@ static int fts3RemoveSegdirEntry( return rc; } -static int fts3AppendToNode( - Blob *pNode, - Blob *pPrev, - const char *zTerm, - int nTerm, - const char *aDoclist, - int nDoclist -){ - int rc = SQLITE_OK; - int bFirst = (pPrev->n==0); - int nPrefix; - int nSuffix; - - blobGrowBuffer(pPrev, nTerm, &rc); - if( rc!=SQLITE_OK ) return rc; - - nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; - memcpy(pPrev->a, zTerm, nTerm); - pPrev->n = nTerm; - - if( bFirst==0 ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); - } - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); - memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); - pNode->n += nSuffix; - - if( aDoclist ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); - memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); - pNode->n += nDoclist; - } - - return SQLITE_OK; -} - static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ pNode->a[0] = (char)iHeight; if( iChild ){ @@ -4025,9 +4139,8 @@ static int fts3TruncateNode( sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ ){ NodeReader reader; /* Reader object */ - Blob prev = {0, 0, 0}; - int rc = SQLITE_OK; - int bStarted = 0; + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ /* Allocate required output space */ @@ -4040,13 +4153,11 @@ static int fts3TruncateNode( rc==SQLITE_OK && reader.aNode; rc = nodeReaderNext(&reader) ){ - if( bStarted==0 ){ + if( pNew->n==0 ){ int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); if( res<0 || (bLeaf==0 && res==0) ) continue; - pNew->a[0] = aNode[0]; fts3StartNode(pNew, (int)aNode[0], reader.iChild); *piBlock = reader.iChild; - bStarted = 1; } rc = fts3AppendToNode( pNew, &prev, reader.term.a, reader.term.n, @@ -4054,7 +4165,7 @@ static int fts3TruncateNode( ); if( rc!=SQLITE_OK ) break; } - if( bStarted==0 ){ + if( pNew->n==0 ){ fts3StartNode(pNew, (int)aNode[0], reader.iChild); *piBlock = reader.iChild; } @@ -4065,6 +4176,15 @@ static int fts3TruncateNode( return rc; } +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ static int fts3TruncateSegment( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ @@ -4151,9 +4271,9 @@ static int fts3TruncateSegment( ** have been duplicated in the output segment. */ static int fts3IncrmergeChomp( - Fts3Table *p, - sqlite3_int64 iAbsLevel, - Fts3MultiSegReader *pCsr + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr /* Chomp all segments opened by this cursor */ ){ int i; int rc = SQLITE_OK; @@ -4190,13 +4310,13 @@ static int fts3IncrmergeChomp( } /* -** Attempt an incremental merge that writes nMerge leaf pages. +** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen two segments at a time. The two -** segments to be merged are the two oldest segments (the ones with -** the smallest index) in the highest level that has at least -** nMin segments. Multiple segment pair merges might occur in -** an attempt to write the quota of nMerge leaf pages. +** Incremental merges happen nMin segments at a time. The two +** segments to be merged are the nMin oldest segments (the ones with +** the smallest indexes) in the highest level that contains at least +** nMin segments. Multiple merges might occur in an attempt to write the +** quota of nMerge leaf blocks. */ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc = SQLITE_OK; /* Return code */ @@ -4236,9 +4356,7 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** the selected absolute level. */ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; rc = fts3IncrmergeCsr(p, iAbsLevel, nMin, pCsr); -sqlite3_log(SQLITE_OK, "%d-way merge from level=%d to level=%d", - nMin, (int)iAbsLevel, (int)iAbsLevel+1 -); + fts3LogMerge(nMin, iAbsLevel); if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter); } diff --git a/manifest b/manifest index 8bb46fe386..925cbf7464 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sinteger\soverflow\sproblems\sthat\scan\soccur\swhen\susing\slarge\slangauge\sid\svalues. -D 2012-03-16T15:54:19.453 +C Fix\svarious\sincorrect\sand\smissing\scomments\sand\sother\sstyle\sissues\sin\sand\saround\sthe\sFTS\sincremental\smerge\scode. +D 2012-03-17T16:56:57.450 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -65,7 +65,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 806632fd0020eed966ab82ea25fe09f1a4c86907 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 1da6d2af6b079cdd74cd2350761182dedb8bd892 +F ext/fts3/fts3Int.h caa745f80405bc0c0a45a2f83e120d5a2c13753c F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 548d034e16ecad0ec5dfd542c48c591564c343d3 +F ext/fts3/fts3_write.c a78deea7b575b244d50eea1fdb9bf228c0dd00bb F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -994,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd -R 3c4572a4f07a3af6855856a4d0059330 +P 3475092cff862080a020d386076d739f0d22c9b2 +R ca4e10aa887c331ab5ee7949ea058bc8 U dan -Z 5aeec06647e7b858688ed8df27a75aed +Z 5217f3a7f5fecf12c3e93b43d9936afd diff --git a/manifest.uuid b/manifest.uuid index b07a988ed8..ecbe0fe2b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3475092cff862080a020d386076d739f0d22c9b2 \ No newline at end of file +7aabb62c8ccbd2b8d216e25226f06e5820dec38a \ No newline at end of file From bbe05203299797220defebb10212be4a7cdb7613 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Mar 2012 14:28:43 +0000 Subject: [PATCH 12/68] Avoid a compiler warning (an incorrect compiler warning, at that) in vs2010. FossilOrigin-Name: 7dd97f12cd268cac1241f0f3e8de94bb629b97c7 --- ext/fts3/fts3.c | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 96905fc14b..04671af24f 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1223,7 +1223,8 @@ static int fts3InitVtab( int j; for(j=0; j Date: Mon, 19 Mar 2012 14:51:19 +0000 Subject: [PATCH 13/68] Suppress harmless compiler warnings on windows in FTS4 and RTREE. FossilOrigin-Name: 4fd68647c8d4b120e04d054617cef31001f44c6f --- ext/fts3/fts3.c | 24 ++++++++++++------------ ext/fts3/fts3_aux.c | 4 ++-- ext/rtree/rtree.c | 8 ++++---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 04671af24f..984712c7b7 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -741,7 +741,7 @@ static void fts3Appendf( static char *fts3QuoteId(char const *zInput){ int nRet; char *zRet; - nRet = 2 + strlen(zInput)*2 + 1; + nRet = 2 + (int)strlen(zInput)*2 + 1; zRet = sqlite3_malloc(nRet); if( zRet ){ int i; @@ -997,7 +997,7 @@ static int fts3ContentColumns( nCol = sqlite3_column_count(pStmt); for(i=0; idoclist.pList = 0; @@ -3820,7 +3820,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aOut; if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); }else{ sqlite3_free(aOut); pPhrase->doclist.pList = 0; @@ -3916,7 +3916,7 @@ void sqlite3Fts3DoclistPrev( iMul = (bDescIdx ? -1 : 1); } - *pnList = pEnd - pNext; + *pnList = (int)(pEnd - pNext); *ppIter = pNext; *piDocid = iDocid; }else{ @@ -3930,7 +3930,7 @@ void sqlite3Fts3DoclistPrev( }else{ char *pSave = p; fts3ReversePoslist(aDoclist, &p); - *pnList = (pSave - p); + *pnList = (int)(pSave - p); } *ppIter = p; } @@ -4348,8 +4348,8 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Expr **ppOr = apOr; fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = pTC-aTC; - nOr = ppOr-apOr; + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); if( rc==SQLITE_OK ){ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); @@ -4421,7 +4421,7 @@ static int fts3EvalNearTrim( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 ); if( res ){ - nNew = (pOut - pPhrase->doclist.pList) - 1; + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; assert( pPhrase->doclist.pList[nNew]=='\0' ); assert( nNew<=pPhrase->doclist.nList && nNew>0 ); memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 89bf3ebf6e..a2bff2e1d3 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -79,9 +79,9 @@ static int fts3auxConnectMethod( } zDb = argv[1]; - nDb = strlen(zDb); + nDb = (int)strlen(zDb); zFts3 = argv[3]; - nFts3 = strlen(zFts3); + nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 4c36f26171..ce76e61f08 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3056,8 +3056,8 @@ static int rtreeInit( sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Allocate the sqlite3_vtab structure */ - nDb = strlen(argv[1]); - nName = strlen(argv[2]); + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); if( !pRtree ){ return SQLITE_NOMEM; @@ -3152,10 +3152,10 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nodeGetCell(&tree, &node, ii, &cell); sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); - nCell = strlen(zCell); + nCell = (int)strlen(zCell); for(jj=0; jj Date: Mon, 19 Mar 2012 14:57:49 +0000 Subject: [PATCH 14/68] Fix one more compiler warning missed by the previous check-in. FossilOrigin-Name: bc03d99a78e90c02b69037e5f5f81537b5a3ac60 --- ext/fts3/fts3.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 984712c7b7..421052b937 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3990,7 +3990,7 @@ static int fts3EvalPhraseNext( } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); - pDL->nList = (pIter - pDL->pList); + pDL->nList = (int)(pIter - pDL->pList); /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was diff --git a/manifest b/manifest index 853c98bd07..02ccade03d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Suppress\sharmless\scompiler\swarnings\son\swindows\sin\sFTS4\sand\sRTREE. -D 2012-03-19T14:51:19.261 +C Fix\sone\smore\scompiler\swarning\smissed\sby\sthe\sprevious\scheck-in. +D 2012-03-19T14:57:49.232 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c cb5b77609ebe3405f482b81a419e12dd00b5d9b5 +F ext/fts3/fts3.c a62e09140c00f9c401efca661e7f2ae9909194f6 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d1d7f964ddee067bcd16a6af4ba7ecf66220056d F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 7dd97f12cd268cac1241f0f3e8de94bb629b97c7 -R 3d835a8bbdb81ada9b824f3c7c84ac79 +P 4fd68647c8d4b120e04d054617cef31001f44c6f +R d0de9386b7862d0d496b567825b3fc4a U drh -Z 908ca5c15e2aee882328fc436d7a8c0f +Z d75a5126aa3d9bafa8a4a6d3f6859d6d diff --git a/manifest.uuid b/manifest.uuid index 9f22b09f51..8ce488b11e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fd68647c8d4b120e04d054617cef31001f44c6f \ No newline at end of file +bc03d99a78e90c02b69037e5f5f81537b5a3ac60 \ No newline at end of file From 0299b40f0fa6ae9de1af5dd1c15cb64fc7797a1a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Mar 2012 17:42:46 +0000 Subject: [PATCH 15/68] Add additional test cases to e_insert.test. Update evidence marks. no changes to core code. FossilOrigin-Name: 036395c0a8e08883b11df025e3da9e2461e4b1eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/util.c | 10 +++++----- test/e_insert.test | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 02ccade03d..50503f5402 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sone\smore\scompiler\swarning\smissed\sby\sthe\sprevious\scheck-in. -D 2012-03-19T14:57:49.232 +C Add\sadditional\stest\scases\sto\se_insert.test.\s\sUpdate\sevidence\smarks.\nno\schanges\sto\score\scode. +D 2012-03-19T17:42:46.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 -F src/util.c 906731099c4397bf8adf3fa90a833355e7472af0 +F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 F src/vdbe.c 32720e873ed0a23e6ee928b676cd995864b984d6 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb @@ -383,7 +383,7 @@ F test/e_dropview.test 583411e470458c5d76148542cfb5a5fa84c8f93e F test/e_expr.test 5489424d3d9a452ac3701cdf4b680ae31a157894 F test/e_fkey.test 057eed81a41a2b21b1790032f4e8aaba0b2b0e17 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 -F test/e_insert.test 92d2dab07366aef112f53af4539e30559f5d35a7 +F test/e_insert.test c6ac239a97cb16dfbd0c16496f8cd871b4068c0c F test/e_reindex.test dfedfc32c5a282b0596c6537cbcd4217fbb1a216 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test f5d4b81205701deacfae42051ae200969c41d2c0 @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 4fd68647c8d4b120e04d054617cef31001f44c6f -R d0de9386b7862d0d496b567825b3fc4a +P bc03d99a78e90c02b69037e5f5f81537b5a3ac60 +R 69afcc9757d71d53b13c043daaf0c06c U drh -Z d75a5126aa3d9bafa8a4a6d3f6859d6d +Z e41b3b1650d5b16145f025deed2994e7 diff --git a/manifest.uuid b/manifest.uuid index 8ce488b11e..0d3f57ac8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc03d99a78e90c02b69037e5f5f81537b5a3ac60 \ No newline at end of file +036395c0a8e08883b11df025e3da9e2461e4b1eb \ No newline at end of file diff --git a/src/util.c b/src/util.c index 325c75aae1..dd3b08ae46 100644 --- a/src/util.c +++ b/src/util.c @@ -216,11 +216,11 @@ int sqlite3Dequote(char *z){ ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. ** -** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows -** applications and extensions to compare the contents of two buffers -** containing UTF-8 strings in a case-independent fashion, using the same -** definition of case independence that SQLite uses internally when -** comparing identifiers. +** IMPLEMENTATION-OF: R-30243-02494 The sqlite3_stricmp() and +** sqlite3_strnicmp() APIs allow applications and extensions to compare +** the contents of two buffers containing UTF-8 strings in a +** case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; diff --git a/test/e_insert.test b/test/e_insert.test index a10d5283de..ac4361f8df 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -50,7 +50,7 @@ proc do_insert_tests {args} { uplevel do_select_tests $args } -# EVIDENCE-OF: R-55375-41353 -- syntax diagram insert-stmt +# EVIDENCE-OF: R-21350-31508 -- syntax diagram insert-stmt # do_insert_tests e_insert-0 { 1 "INSERT INTO a1 DEFAULT VALUES" {} @@ -123,6 +123,20 @@ do_insert_tests e_insert-0 { 68 "INSERT OR IGNORE INTO a1 (b, a) SELECT c, b FROM a2" {} 69 "REPLACE INTO a1 (b, a) SELECT c, b FROM a2" {} 70 "REPLACE INTO main.a1 (b, a) SELECT c, b FROM a2" {} + 71 "INSERT INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 72 "INSERT INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 73 "INSERT OR ROLLBACK INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 74 "INSERT OR ROLLBACK INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 75 "INSERT OR ABORT INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 76 "INSERT OR ABORT INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 77 "INSERT OR REPLACE INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 78 "INSERT OR REPLACE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 79 "INSERT OR FAIL INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 80 "INSERT OR FAIL INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 81 "INSERT OR FAIL INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 82 "INSERT OR IGNORE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 83 "REPLACE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 84 "REPLACE INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} } delete_all_data From 82ebc2a098bbe4cb66e18b50d65802f6263a41f1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Mar 2012 03:10:51 +0000 Subject: [PATCH 16/68] Fix out-of-bounds array references in the "echo" virtual table module used for testing. No changes to the SQLite core. FossilOrigin-Name: 7b449b301ea03295262b8d572b02625e4b39cfa5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test8.c | 16 ++++++---------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 50503f5402..e616bb0737 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sadditional\stest\scases\sto\se_insert.test.\s\sUpdate\sevidence\smarks.\nno\schanges\sto\score\scode. -D 2012-03-19T17:42:46.646 +C Fix\sout-of-bounds\sarray\sreferences\sin\sthe\s"echo"\svirtual\stable\smodule\nused\sfor\stesting.\s\sNo\schanges\sto\sthe\sSQLite\score. +D 2012-03-20T03:10:51.841 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c 846ed1ed2f470de9b1e205fe3878a12e237b3e19 F src/test7.c 2e0781754905c8adc3268d8f0967e7633af58843 -F src/test8.c 99f70341d6ec480313775127f4cd14b4a47db557 +F src/test8.c 61b41d79509a479dec1ac32b6d4209b27c4b1ba5 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bc03d99a78e90c02b69037e5f5f81537b5a3ac60 -R 69afcc9757d71d53b13c043daaf0c06c +P 036395c0a8e08883b11df025e3da9e2461e4b1eb +R 4ef201d6122aecff56466ef062fb4020 U drh -Z e41b3b1650d5b16145f025deed2994e7 +Z 262756c9236d4e1eee4ee8920cc99860 diff --git a/manifest.uuid b/manifest.uuid index 0d3f57ac8d..ac6be16a19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -036395c0a8e08883b11df025e3da9e2461e4b1eb \ No newline at end of file +7b449b301ea03295262b8d572b02625e4b39cfa5 \ No newline at end of file diff --git a/src/test8.c b/src/test8.c index 283d790b7d..80e7adaf2c 100644 --- a/src/test8.c +++ b/src/test8.c @@ -831,13 +831,10 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( !isIgnoreUsable && !pConstraint->usable ) continue; iCol = pConstraint->iColumn; - if( pVtab->aIndex[iCol] || iCol<0 ){ - char *zCol = pVtab->aCol[iCol]; + if( iCol<0 || pVtab->aIndex[iCol] ){ + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zOp = 0; useIdx = 1; - if( iCol<0 ){ - zCol = "rowid"; - } switch( pConstraint->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: zOp = "="; break; @@ -870,13 +867,12 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** on a column that this virtual table has an index for, then consume ** the ORDER BY clause. */ - if( pIdxInfo->nOrderBy==1 && pVtab->aIndex[pIdxInfo->aOrderBy->iColumn] ){ + if( pIdxInfo->nOrderBy==1 && ( + pIdxInfo->aOrderBy->iColumn<0 || + pVtab->aIndex[pIdxInfo->aOrderBy->iColumn]) ){ int iCol = pIdxInfo->aOrderBy->iColumn; - char *zCol = pVtab->aCol[iCol]; + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zDir = pIdxInfo->aOrderBy->desc?"DESC":"ASC"; - if( iCol<0 ){ - zCol = "rowid"; - } zNew = sqlite3_mprintf(" ORDER BY %s %s", zCol, zDir); string_concat(&zQuery, zNew, 1, &rc); pIdxInfo->orderByConsumed = 1; From 7b7c359fe6802552af23cb5eb9a60666c37e878b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Mar 2012 11:35:50 +0000 Subject: [PATCH 17/68] Version 3.7.11 FossilOrigin-Name: 00bb9c9ce4f465e6ac321ced2a9d0062dc364669 --- manifest | 9 +++++---- manifest.uuid | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index e616bb0737..ddee8a5677 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sout-of-bounds\sarray\sreferences\sin\sthe\s"echo"\svirtual\stable\smodule\nused\sfor\stesting.\s\sNo\schanges\sto\sthe\sSQLite\score. -D 2012-03-20T03:10:51.841 +C Version\s3.7.11 +D 2012-03-20T11:35:50.522 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -992,7 +992,8 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 036395c0a8e08883b11df025e3da9e2461e4b1eb +P 7b449b301ea03295262b8d572b02625e4b39cfa5 R 4ef201d6122aecff56466ef062fb4020 +T +sym-version-3.7.11 * U drh -Z 262756c9236d4e1eee4ee8920cc99860 +Z 8740abf5f4801e70716f6cff3569cf87 diff --git a/manifest.uuid b/manifest.uuid index ac6be16a19..bfcb78cfbd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b449b301ea03295262b8d572b02625e4b39cfa5 \ No newline at end of file +00bb9c9ce4f465e6ac321ced2a9d0062dc364669 \ No newline at end of file From d878cab51a7563e11ae82ea824fc5941b7cdf2dc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Mar 2012 15:10:42 +0000 Subject: [PATCH 18/68] Remove the _SafeInit() entry points from the TCL interface. They have long been no-ops. Removing them completely avoids confusion as to why they don't work. FossilOrigin-Name: 0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/tclsqlite.c | 12 ++++-------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ddee8a5677..60db4f1f2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.7.11 -D 2012-03-20T11:35:50.522 +C Remove\sthe\s_SafeInit()\sentry\spoints\sfrom\sthe\sTCL\sinterface.\s\sThey\shave\slong\nbeen\sno-ops.\s\sRemoving\sthem\scompletely\savoids\sconfusion\sas\sto\swhy\sthey\ndon't\swork. +D 2012-03-20T15:10:42.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -188,7 +188,7 @@ F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 2aeb69958965dad0842d5ea1456f1a958ef296e6 +F src/tclsqlite.c 086dfdd72e5892de223968a258e1ccbd9693e717 F src/test1.c 07f30e8714bab94d8de8a88865d9c59bc512a1a8 F src/test2.c 711555927f1f7e8db9aab86b512bc6934a774abe F src/test3.c 91d3f1a09cfae3533ef17d8b484a160f3d1f1a21 @@ -992,8 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 7b449b301ea03295262b8d572b02625e4b39cfa5 -R 4ef201d6122aecff56466ef062fb4020 -T +sym-version-3.7.11 * +P 00bb9c9ce4f465e6ac321ced2a9d0062dc364669 +R 0bae8756351ce6c993966becd439fadf U drh -Z 8740abf5f4801e70716f6cff3569cf87 +Z d1e2a2d0a2cbd6a9676ec1f5c607220d diff --git a/manifest.uuid b/manifest.uuid index bfcb78cfbd..f2c6d71cbb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00bb9c9ce4f465e6ac321ced2a9d0062dc364669 \ No newline at end of file +0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 967b1a004f..9161088e08 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3108,23 +3108,19 @@ EXTERN int Sqlite3_Init(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +/* Because it accesses the file-system and uses persistent state, SQLite +** is not considered appropriate for safe interpreters. Hence, we deliberately +** omit the _SafeInit() interfaces. +*/ #ifndef SQLITE_3_SUFFIX_ONLY int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #endif #ifdef TCLSH From 74f47e1d311e406d15edb6a791ac67716c3cf248 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Mar 2012 14:34:23 +0000 Subject: [PATCH 19/68] Add fts4merge3.test, for testing that older versions of FTS4 may interoperate with incr-merge capable versions. FossilOrigin-Name: 903ec5126dd981da6d7bab45c568f34b99446159 --- manifest | 18 +++++---- manifest.uuid | 2 +- test/backcompat.test | 30 ++------------ test/bc_common.tcl | 76 +++++++++++++++++++++++++++++++++++ test/fts3_common.tcl | 38 ++++++++++++++++-- test/fts4merge3.test | 96 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 39 deletions(-) create mode 100644 test/bc_common.tcl create mode 100644 test/fts4merge3.test diff --git a/manifest b/manifest index 7d1a25eb35..13ef1aed2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthe\sfts4-incr-merge\sbranch. -D 2012-03-20T17:04:17.255 +C Add\sfts4merge3.test,\sfor\stesting\sthat\solder\sversions\sof\sFTS4\smay\sinteroperate\swith\sincr-merge\scapable\sversions. +D 2012-03-21T14:34:23.781 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,13 +288,14 @@ F test/autoindex1.test 058d0b331ae6840a61bbee910d8cbae27bfd5991 F test/autovacuum.test fcaf4616ae5bb18098db1cb36262565e5c841c3c F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 -F test/backcompat.test 71eeb75ea567c060774c4e8db4b0e703f21c7677 +F test/backcompat.test 94778872ed9f6158ba1b8534264d3413ae7e27f8 F test/backup.test 717346db953e9e435c2a94916e4af177330d60d3 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f +F test/bc_common.tcl df4c51ae0de0cb414a18027be2abb2bd7693ce7a F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 F test/bigfile.test 82dfe93ee7eb9e2e05641afa2b39ffd947a92ff1 F test/bigfile2.test 852f948cb492aadab45b58f4d2f3b0832a115cb0 @@ -448,7 +449,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 -F test/fts3_common.tcl d65de7e21c2b933bf17152830eb924356f197c8f +F test/fts3_common.tcl 52423d0f15bca1d1d2f2b432f0542c4036d655c7 F test/fts3aa.test 909d5f530d30a8e36b9328d67285eae6537c79c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 @@ -499,6 +500,7 @@ F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test 1c3389a3be18498934baf76afe67663d8b4b5e42 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 +F test/fts4merge3.test 2d2008772f45afc617fc34d10bcafbc8de1ca2ae F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -994,7 +996,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 7aabb62c8ccbd2b8d216e25226f06e5820dec38a 0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b -R 58077ce5e1cbaecc58d15ee0d3722b1c -U drh -Z 8e670a9384f5fc62996e59150aae6127 +P f61d5fb0281381228eb1a12a233bacaeb26b12a3 +R 0ef159b3f0bfab79746087f966c7f8a3 +U dan +Z 5c824893eaa42ecaf0e347109e073f8a diff --git a/manifest.uuid b/manifest.uuid index 88dba24d9f..4039183245 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f61d5fb0281381228eb1a12a233bacaeb26b12a3 \ No newline at end of file +903ec5126dd981da6d7bab45c568f34b99446159 \ No newline at end of file diff --git a/test/backcompat.test b/test/backcompat.test index e8e2f61581..6a0f91ae97 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -27,37 +27,13 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl +source $testdir/bc_common.tcl db close -# Search for binaries to test against. Any executable files that match -# our naming convention are assumed to be testfixture binaries to test -# against. -# -set binaries [list] -set pattern "[file tail [info nameofexec]]?*" -if {$tcl_platform(platform)=="windows"} { - set pattern [string map {\.exe {}} $pattern] -} -foreach file [glob -nocomplain $pattern] { - if {[file executable $file] && [file isfile $file]} {lappend binaries $file} -} -if {[llength $binaries]==0} { - puts "WARNING: No historical binaries to test against." - puts "WARNING: No backwards-compatibility tests have been run." +if {"" == [bc_find_binaries backcompat.test]} { finish_test return } -proc get_version {binary} { - set chan [launch_testfixture $binary] - set v [testfixture $chan { sqlite3 -version }] - close $chan - set v -} -foreach bin $binaries { - puts -nonewline "Testing against $bin - " - flush stdout - puts "version [get_version $bin]" -} proc do_backcompat_test {rv bin1 bin2 script} { @@ -93,7 +69,7 @@ proc do_backcompat_test {rv bin1 bin2 script} { array set ::incompatible [list] proc do_allbackcompat_test {script} { - foreach bin $::binaries { + foreach bin $::BC(binaries) { set nErr [set_test_counter errors] foreach dir {0 1} { diff --git a/test/bc_common.tcl b/test/bc_common.tcl new file mode 100644 index 0000000000..8bba0245a5 --- /dev/null +++ b/test/bc_common.tcl @@ -0,0 +1,76 @@ + + + +proc bc_find_binaries {zCaption} { + # Search for binaries to test against. Any executable files that match + # our naming convention are assumed to be testfixture binaries to test + # against. + # + set binaries [list] + set pattern "[file tail [info nameofexec]]?*" + if {$::tcl_platform(platform)=="windows"} { + set pattern [string map {\.exe {}} $pattern] + } + foreach file [glob -nocomplain $pattern] { + if {[file executable $file] && [file isfile $file]} {lappend binaries $file} + } + + if {[llength $binaries]==0} { + puts "WARNING: No historical binaries to test against." + puts "WARNING: Omitting backwards-compatibility tests $zFile" + } + + foreach bin $binaries { + puts -nonewline "Testing against $bin - " + flush stdout + puts "version [get_version $bin]" + } + + set ::BC(binaries) $binaries + return $binaries +} + +proc get_version {binary} { + set chan [launch_testfixture $binary] + set v [testfixture $chan { sqlite3 -version }] + close $chan + set v +} + +proc do_bc_test {bin script} { + + forcedelete test.db + set ::bc_chan [launch_testfixture $bin] + + proc code1 {tcl} { uplevel #0 $tcl } + proc code2 {tcl} { testfixture $::bc_chan $tcl } + proc sql1 sql { code1 [list db eval $sql] } + proc sql2 sql { code2 [list db eval $sql] } + + code1 { sqlite3 db test.db } + code2 { sqlite3 db test.db } + + set bintag [string map {testfixture {}} $bin] + set bintag [string map {\.exe {}} $bintag] + if {$bintag == ""} {set bintag self} + set saved_prefix $::testprefix + append ::testprefix ".$bintag" + + uplevel $script + + set ::testprefix $saved_prefix + + catch { code1 { db close } } + catch { code2 { db close } } + catch { close $::bc_chan } +} + +proc do_all_bc_test {script} { + foreach bin $::BC(binaries) { + uplevel [list do_bc_test $bin $script] + } +} + + + + diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index f031f927d6..d36ca10989 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -14,6 +14,35 @@ # to use Tcl. # +#------------------------------------------------------------------------- +# INSTRUCTIONS +# +# The following commands are available: +# +# fts3_build_db_1 N +# Using database handle [db] create an FTS4 table named t1 and populate +# it with N rows of data. N must be less than 10,000. Refer to the +# header comments above the proc implementation below for details. +# +# fts3_build_db_2 N +# Using database handle [db] create an FTS4 table named t2 and populate +# it with N rows of data. N must be less than 100,000. Refer to the +# header comments above the proc implementation below for details. +# +# fts3_integrity_check TBL +# TBL must be an FTS table in the database currently opened by handle +# [db]. This proc loads and tokenizes all documents within the table, +# then checks that the current contents of the FTS index matches the +# results. +# +# fts3_terms TBL WHERE +# Todo. +# +# fts3_doclist TBL TERM WHERE +# Todo. +# +# +# #------------------------------------------------------------------------- # USAGE: fts3_build_db_1 N @@ -51,16 +80,19 @@ proc fts3_build_db_1 {n} { } #------------------------------------------------------------------------- -# USAGE: fts3_build_db_2 N +# USAGE: fts3_build_db_2 N ARGS # # Build a sample FTS table in the database opened by database connection # [db]. The name of the new table is "t2". # -proc fts3_build_db_2 {n} { +proc fts3_build_db_2 {n args} { if {$n > 100000} {error "n must be <= 100000"} - db eval { CREATE VIRTUAL TABLE t2 USING fts4 } + set sql "CREATE VIRTUAL TABLE t2 USING fts4(content" + foreach a $args { append sql ", " $a } + append sql ")" + db eval $sql set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z ""] diff --git a/test/fts4merge3.test b/test/fts4merge3.test new file mode 100644 index 0000000000..30ea92cc8d --- /dev/null +++ b/test/fts4merge3.test @@ -0,0 +1,96 @@ +# 2012 March 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the incremental merge function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +source $testdir/lock_common.tcl +source $testdir/bc_common.tcl + +set ::testprefix fts4merge3 + +if {"" == [bc_find_binaries backcompat.test]} { + finish_test + return +} + +do_all_bc_test { + + sql2 { PRAGMA page_size = 512 } + if { 0==[catch { sql2 { CREATE VIRTUAL TABLE x USING fts4 } } ] } { + + # Build a large database. + set msg "this takes around 12 seconds" + do_test "1.1 ($msg)" { fts3_build_db_2 20000 } {} + + # Run some queries on it, using the old and new versions. + do_test 1.2 { sql1 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485} + do_test 1.3 { sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485} + + do_test 1.4 { sql2 "PRAGMA page_count" } {1286} + do_test 1.5 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } [list 0 15 1 1 2 14 3 4] + + # Run some incr-merge operations on the db. + for {set i 0} {$i<10} {incr i} { + do_test 1.6.$i.1 { sql1 { INSERT INTO t2(t2) VALUES('merge=2,2') } } {} + do_test 1.6.$i.2 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485} + } + + do_test 1.7 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } [list 0 15 1 1 2 14 3 4 4 1] + + # Using the old connection, insert many rows. + do_test 1.8 { + for {set i 0} {$i < 1500} {incr i} { + sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i" + } + } {} + + do_test 1.9 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } {0 11 1 15 2 3 3 5 4 1} + + # Run a big incr-merge operation on the db. + do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {} + do_test 1.11 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485} + + do_test 1.12 { + for {set i 0} {$i < 1500} {incr i} { + sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i" + } + } {} + do_test 1.13 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485 22985} + + do_test 1.14 { + sql2 "INSERT INTO t2(t2) VALUES('optimize')" + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485 22985} + + do_test 1.15 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } {5 1} + } +} + + +finish_test From bf92ec0ce349f754d4b8df75f415138fa915cd43 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 22 Mar 2012 12:50:34 +0000 Subject: [PATCH 20/68] Always quote the names of tables in the output of the shell's ".dump" command, even if the name is pure alphabetic text, in case the name is a keyword. FossilOrigin-Name: 638b71150281a211f89b4057b0d5d32d3fbcf323 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 60db4f1f2e..a8b57dd4e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\s_SafeInit()\sentry\spoints\sfrom\sthe\sTCL\sinterface.\s\sThey\shave\slong\nbeen\sno-ops.\s\sRemoving\sthem\scompletely\savoids\sconfusion\sas\sto\swhy\sthey\ndon't\swork. -D 2012-03-20T15:10:42.442 +C Always\squote\sthe\snames\sof\stables\sin\sthe\soutput\sof\sthe\sshell's\s".dump"\scommand,\neven\sif\sthe\sname\sis\spure\salphabetic\stext,\sin\scase\sthe\sname\sis\sa\skeyword. +D 2012-03-22T12:50:34.137 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 -F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e +F src/shell.c 55e09ef7126768b940427d95dc0a8cb7138e95da F src/sqlite.h.in 6af2d92925bfed3dfd2c022fef48469762ccd435 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 00bb9c9ce4f465e6ac321ced2a9d0062dc364669 -R 0bae8756351ce6c993966becd439fadf +P 0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b +R 76825aab78a4aec4e0f8b3a0fc8144af U drh -Z d1e2a2d0a2cbd6a9676ec1f5c607220d +Z cbcc6857e5f2528670826efd0ad94a0c diff --git a/manifest.uuid b/manifest.uuid index f2c6d71cbb..f6c5dfb8e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b \ No newline at end of file +638b71150281a211f89b4057b0d5d32d3fbcf323 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index d3ddfa9536..9ea5ced282 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1287,7 +1287,6 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ char *zTableInfo = 0; char *zTmp = 0; int nRow = 0; - int kk; zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); zTableInfo = appendText(zTableInfo, zTable, '"'); @@ -1300,12 +1299,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); - if( !isalpha(zTable[0]) ){ - kk = 0; - }else{ - for(kk=1; isalnum(zTable[kk]); kk++){} - } - zTmp = appendText(zTmp, zTable, zTable[kk] ? '"' : 0); + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + zTmp = appendText(zTmp, zTable, '"'); if( zTmp ){ zSelect = appendText(zSelect, zTmp, '\''); } From e81eaec754e5ab2667d6cdb9ce0f4d6fcfd70b77 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 22 Mar 2012 16:48:12 +0000 Subject: [PATCH 21/68] Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off. FossilOrigin-Name: ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb --- ext/fts3/fts3_write.c | 317 ++++++++++++++++++++++++++++++------------ manifest | 14 +- manifest.uuid | 2 +- test/fts4merge.test | 59 ++++++-- 4 files changed, 283 insertions(+), 109 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1218814b4a..5b8fcc2ab2 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -66,6 +66,12 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic @@ -243,8 +249,8 @@ struct SegmentNode { #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_DOCTOTAL 22 -#define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 @@ -305,8 +311,8 @@ static int fts3SqlStmt( /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", /* 24 */ "", /* 25 */ "", @@ -318,7 +324,7 @@ static int fts3SqlStmt( ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>?" +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" " ORDER BY (level %% 1024) DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment @@ -392,20 +398,15 @@ static int fts3SqlStmt( static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ - int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ int rc; /* Return code */ - assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); - - rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); if( rc==SQLITE_OK ){ - if( eStmt==SQL_SELECT_DOCSIZE ){ - sqlite3_bind_int64(pStmt, 1, iDocid); - } + sqlite3_bind_int64(pStmt, 1, iDocid); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); @@ -424,7 +425,21 @@ int sqlite3Fts3SelectDoctotal( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); + sqlite3_stmt *pStmt = 0; + int rc; + rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)!=SQLITE_ROW + || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB + ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + } + } + *ppStmt = pStmt; + return rc; } int sqlite3Fts3SelectDocsize( @@ -432,7 +447,7 @@ int sqlite3Fts3SelectDocsize( sqlite3_int64 iDocid, /* Docid to read size data for */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); + return fts3SelectDocsize(pTab, iDocid, ppStmt); } /* @@ -3094,12 +3109,13 @@ static void fts3UpdateDocTotals( return; } pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), @@ -3123,13 +3139,14 @@ static void fts3UpdateDocTotals( a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } - sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); @@ -3277,13 +3294,13 @@ static int fts3IncrmergeCsr( rc = SQLITE_NOMEM; }else{ memset(pCsr->apSegment, 0, nByte); - pCsr->nSegment = nSeg; rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); } if( rc==SQLITE_OK ){ int i; int rc2; sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] ); + pCsr->nSegment++; } rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = rc2; @@ -3970,11 +3988,37 @@ static int fts3IncrmergeLoad( return rc; } +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + /* -** Either allocate an output segment or locate an existing appendable -** output segment to append to. An "appendable" output segment is -** slightly different to a normal one, as the required range of keys in -** the %_segments table must be allocated up front. +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. ** ** In the %_segdir table, a segment is defined by the values in three ** columns: @@ -4001,33 +4045,15 @@ static int fts3IncrmergeLoad( static int fts3IncrmergeWriter( Fts3Table *p, /* Fts3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ IncrmergeWriter *pWriter /* Populate this object */ ){ int rc; /* Return Code */ int i; /* Iterator variable */ int nLeafEst; /* Blocks allocated for leaf nodes */ - int iIdx; /* Index of output segment */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ - sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ - const char *zKey = pCsr->zTerm; /* First key to be appended to output */ - int nKey = pCsr->nTerm; /* Size of zKey in bytes */ - - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); - sqlite3_step(pOutputIdx); - iIdx = sqlite3_column_int(pOutputIdx, 0) - 1; - rc = sqlite3_reset(pOutputIdx); - } - if( rc!=SQLITE_OK ) return rc; - - assert( zKey ); - assert( pWriter->nLeafEst==0 ); - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx, zKey, nKey, pWriter); - if( rc!=SQLITE_OK || pWriter->nLeafEst ) return rc; - iIdx++; /* Calculate nLeafEst. */ rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); @@ -4190,7 +4216,7 @@ static int fts3TruncateSegment( sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ int iIdx, /* Index within level of segment to modify */ const char *zTerm, /* Remove terms smaller than this */ - int nTerm /* Number of bytes in buffer zTerm */ + int nTerm /* Number of bytes in buffer zTerm */ ){ int rc = SQLITE_OK; /* Return code */ Blob root = {0,0,0}; /* New root page image */ @@ -4198,22 +4224,22 @@ static int fts3TruncateSegment( sqlite3_int64 iBlock = 0; /* Block id */ sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ - int rc2; /* sqlite3_reset() return code */ sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ - assert( p->aStmt[SQL_SELECT_SEGDIR] ); - pFetch = p->aStmt[SQL_SELECT_SEGDIR]; - - sqlite3_bind_int64(pFetch, 1, iAbsLevel); - sqlite3_bind_int(pFetch, 2, iIdx); - if( SQLITE_ROW==sqlite3_step(pFetch) ){ - const char *aRoot = sqlite3_column_blob(pFetch, 4); - int nRoot = sqlite3_column_bytes(pFetch, 4); - iOldStart = sqlite3_column_int64(pFetch, 1); - rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; } - rc2 = sqlite3_reset(pFetch); - if( rc==SQLITE_OK ) rc = rc2; while( rc==SQLITE_OK && iBlock ){ char *aBlock = 0; @@ -4273,9 +4299,11 @@ static int fts3TruncateSegment( static int fts3IncrmergeChomp( Fts3Table *p, /* FTS table handle */ sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ - Fts3MultiSegReader *pCsr /* Chomp all segments opened by this cursor */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ ){ int i; + int nRem = 0; int rc = SQLITE_OK; for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ @@ -4296,6 +4324,7 @@ static int fts3IncrmergeChomp( if( rc==SQLITE_OK ){ rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); } + *pnRem = 0; }else{ /* The incremental merge did not copy all the data from this ** segment to the upper level. The segment is modified in place @@ -4303,9 +4332,76 @@ static int fts3IncrmergeChomp( const char *zTerm = pSeg->zTerm; int nTerm = pSeg->nTerm; rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; } } + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to read input data from */ + int nMerge /* Number of segments to merge */ +){ + char aBlob[FTS3_VARINT_MAX * 2]; + int nBlob = 0; + int rc; + sqlite3_stmt *pReplace = 0; + + assert( p->bHasStat ); + nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel); + nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge); + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. +** +** The incr-merge hint, if one exists, is stored in the rowid==1 row of +** the %_stat table. +*/ +static int fts3IncrmergeHintLoad( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 *piAbsLevel, /* Absolute level to read input data from */ + int *pnMerge /* Number of segments to merge */ +){ + sqlite3_stmt *pSelect = 0; + int rc; + + *pnMerge = 0; + *piAbsLevel = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + int i; + char aBlob[FTS3_VARINT_MAX * 2]; + memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint)); + i = sqlite3Fts3GetVarint(aBlob, piAbsLevel); + sqlite3Fts3GetVarint32(&aBlob[i], pnMerge); + } + } + rc = sqlite3_reset(pSelect); + } + return rc; } @@ -4319,31 +4415,48 @@ static int fts3IncrmergeChomp( ** quota of nMerge leaf blocks. */ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ - int rc = SQLITE_OK; /* Return code */ + int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ + int bUseHint = 1; /* True if hint has not yet been attempted */ + sqlite3_int64 iHintAbsLevel = 0;/* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ assert( nMin>=2 ); + rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg); + if( nHintSeg==0 ) bUseHint = 0; + while( rc==SQLITE_OK && nRem>0 ){ - sqlite3_int64 iAbsLevel; /* Absolute level number to work on */ - sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ IncrmergeWriter *pWriter; /* Writer object */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - /* Determine which level to merge segments from. Any level, from any - ** prefix or language index may be selected. Stack variable iAbsLevel - ** is set to the absolute level number of the level to merge from. */ - rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); - if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ - /* There are no levels with nMin or more segments. Or an error has - ** occurred. Either way, exit early. */ - return sqlite3_reset(pFindLevel); + if( bUseHint ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + }else{ + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + + /* Determine which level to merge segments from. Any level, from any + ** prefix or language index may be selected. Stack variable iAbsLevel + ** is set to the absolute level number of the level to merge from. */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ + /* There are no levels with nMin or more segments. Or an error has + ** occurred. Either way, exit early. */ + rc = sqlite3_reset(pFindLevel); + iAbsLevel = 0; + nSeg = 0; + break; + } + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + sqlite3_reset(pFindLevel); } - iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - sqlite3_reset(pFindLevel); /* Allocate space for the cursor, filter and writer objects */ pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); @@ -4352,36 +4465,56 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; - /* Open a cursor to iterate through the contents of indexes 0 and 1 of - ** the selected absolute level. */ + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; - rc = fts3IncrmergeCsr(p, iAbsLevel, nMin, pCsr); - fts3LogMerge(nMin, iAbsLevel); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter); - } - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr, pWriter); - if( rc==SQLITE_OK ){ - do { - rc = fts3IncrmergeAppend(p, pWriter, pCsr); - if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); - if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; - }while( rc==SQLITE_ROW ); + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + if( pCsr->nSegment==nSeg && SQLITE_OK==rc + && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + if( rc==SQLITE_OK ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } } - } - fts3IncrmergeRelease(p, pWriter, &rc); - nRem -= (1 + pWriter->nWork); - /* Update or delete the input segments */ - if( rc==SQLITE_OK ){ - rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr); + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + } + } + + fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); sqlite3_free(pWriter); + bUseHint = 0; + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){ + rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg); } return rc; diff --git a/manifest b/manifest index 13ef1aed2c..7fb243a8e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfts4merge3.test,\sfor\stesting\sthat\solder\sversions\sof\sFTS4\smay\sinteroperate\swith\sincr-merge\scapable\sversions. -D 2012-03-21T14:34:23.781 +C Following\san\sincr-merge\soperation\sthat\sdoes\snot\scompletely\sconsume\sits\sinput\ssegments,\sstore\scontext\sin\sthe\srowid==1\srow\sof\sthe\s%_stat\stable\sthat\sallows\sthe\snext\sincr-merge\sto\spick\sup\swhere\sthe\sprevious\sleft\soff. +D 2012-03-22T16:48:12.328 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c a78deea7b575b244d50eea1fdb9bf228c0dd00bb +F ext/fts3/fts3_write.c 62ada731e3beef4977386a91240c5f80a1db884e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -498,7 +498,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 1c3389a3be18498934baf76afe67663d8b4b5e42 +F test/fts4merge.test 8e092654ab874afcc4a25f2255497eeb37da74e9 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 2d2008772f45afc617fc34d10bcafbc8de1ca2ae F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -996,7 +996,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f61d5fb0281381228eb1a12a233bacaeb26b12a3 -R 0ef159b3f0bfab79746087f966c7f8a3 +P 903ec5126dd981da6d7bab45c568f34b99446159 +R 36f0dde084aa3ea3b70c7d9b0328ed18 U dan -Z 5c824893eaa42ecaf0e347109e073f8a +Z c1efcc23c92abcdb78f85442afae1924 diff --git a/manifest.uuid b/manifest.uuid index 4039183245..2894e9a6a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -903ec5126dd981da6d7bab45c568f34b99446159 \ No newline at end of file +ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 9f7e05b0ed..91f3e04f8b 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -63,9 +63,7 @@ for {set i 0} {$i<100} {incr i} { do_execsql_test 1.5 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level } { - 0 {0 1 2 3} - 1 {0 1 2} - 2 0 + 2 {0 1} 3 0 } @@ -114,12 +112,55 @@ do_execsql_test 3.3 { SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level } { 0 0 - 1 {0 1} - 2 0 - 3 {0 1} - 4 {0 1} - 5 0 + 2 0 + 3 0 + 4 0 + 6 0 } -finish_test +#------------------------------------------------------------------------- +# Test cases 4.* +# +reset_db +do_execsql_test 4.1 { + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t4 USING fts4; + PRAGMA main.page_size; +} {512} +do_test 4.2 { + foreach x {a c b d e f g h i j k l m n o p} { + execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" + } + execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} +} {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} + +foreach {tn expect} { + 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" + 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" + 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" + 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" + 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" + 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" + 7 "0 {0 1 2 3 4 5 6 7} 1 0" + 8 "0 {0 1 2 3 4 5 6} 1 0" + 9 "0 {0 1 2 3 4 5} 1 0" +} { + do_execsql_test 4.3.$tn { + INSERT INTO t4(t4) VALUES('merge=1,16'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; + } $expect +} + +do_execsql_test 4.4.1 { + SELECT quote(value) FROM t4_stat WHERE rowid=1 +} {X'0006'} + +do_execsql_test 4.4.2 { + DELETE FROM t4_stat WHERE rowid=1; + INSERT INTO t4(t4) VALUES('merge=1,12'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; +} "0 {0 1 2 3 4 5} 1 0" + + +finish_test From 185c1fb02eb1c0ef5f1ed2955f951c5f50dbb5a1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 22 Mar 2012 17:48:00 +0000 Subject: [PATCH 22/68] Add test cases to fts4merge.test. FossilOrigin-Name: ecab2083334dcdde24a3c56864114979b7a6f25a --- manifest | 12 +++--- manifest.uuid | 2 +- test/fts4merge.test | 93 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7fb243a8e3..67f50b5a99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Following\san\sincr-merge\soperation\sthat\sdoes\snot\scompletely\sconsume\sits\sinput\ssegments,\sstore\scontext\sin\sthe\srowid==1\srow\sof\sthe\s%_stat\stable\sthat\sallows\sthe\snext\sincr-merge\sto\spick\sup\swhere\sthe\sprevious\sleft\soff. -D 2012-03-22T16:48:12.328 +C Add\stest\scases\sto\sfts4merge.test. +D 2012-03-22T17:48:00.732 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -498,7 +498,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 8e092654ab874afcc4a25f2255497eeb37da74e9 +F test/fts4merge.test 663607bef5575027295d0b7168370ccd0bae6522 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 2d2008772f45afc617fc34d10bcafbc8de1ca2ae F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -996,7 +996,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 903ec5126dd981da6d7bab45c568f34b99446159 -R 36f0dde084aa3ea3b70c7d9b0328ed18 +P ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb +R ad6c9794089ec45a4e493d1d79b6555f U dan -Z c1efcc23c92abcdb78f85442afae1924 +Z 8189e23423b7ab0255c97e908b40cc3e diff --git a/manifest.uuid b/manifest.uuid index 2894e9a6a2..cbb9ac7b02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb \ No newline at end of file +ecab2083334dcdde24a3c56864114979b7a6f25a \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 91f3e04f8b..5d3368e528 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -162,5 +162,98 @@ do_execsql_test 4.4.2 { SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; } "0 {0 1 2 3 4 5} 1 0" +#------------------------------------------------------------------------- +# Test cases 5.* +# +# Test that if a crisis-merge occurs that disrupts an ongoing incremental +# merge, the next call to "merge=A,B" identifies this and starts a new +# incremental merge. There are two scenarios: +# +# * There are less segments on the input level that the disrupted +# incremental merge operated on, or +# +# * Sufficient segments exist on the input level but the segments +# contain keys smaller than the largest key in the potential output +# segment. +# +do_test 5.1 { + reset_db + fts3_build_db_1 1000 +} {} + +do_execsql_test 5.2 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} + +do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('merge=1,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3} +} + +do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_test 5.5 { + foreach docid [execsql {SELECT docid FROM t1}] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } +} {} + +do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} + +do_execsql_test 5.7 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} + 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 2 {0 1 2 3 4 5 6 7} + X'0104' +} + +do_execsql_test 5.8 { + INSERT INTO t1(t1) VALUES('merge=1,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} + 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 2 {0 1 2 3 4 5 6 7} + 3 {0} + X'0204' +} + +do_test 5.9 { + set L [expr 16*16*8 + 16*4 + 1] + foreach docid [execsql { + SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L + }] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } +} {} + +do_execsql_test 5.10 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 0 1 0 2 0 3 {0 1} + X'0204' +} + +do_execsql_test 5.11 { + INSERT INTO t1(t1) VALUES('merge=10,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 0 1 0 2 0 3 {0 1} + X'0000' +} + finish_test From 40c21432b7937ab3b79029a706b770182d6b4fd7 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 11:07:22 +0000 Subject: [PATCH 23/68] Update a couple of existing test cases. FossilOrigin-Name: dcb8fa0f77a44250df0e8c4f6cfb9f6e181982d7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts4merge3.test | 2 +- test/trace2.test | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 67f50b5a99..508b8682ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\sfts4merge.test. -D 2012-03-22T17:48:00.732 +C Update\sa\scouple\sof\sexisting\stest\scases. +D 2012-03-23T11:07:22.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -500,7 +500,7 @@ F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test 663607bef5575027295d0b7168370ccd0bae6522 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 -F test/fts4merge3.test 2d2008772f45afc617fc34d10bcafbc8de1ca2ae +F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -859,7 +859,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 95f6d519f534b171b8d3d1aacbb03931b4443fb4 +F test/trace2.test 9e4c0b3d4a85494acf1c1e077276d1e934100ce4 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 @@ -996,7 +996,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb -R ad6c9794089ec45a4e493d1d79b6555f +P ecab2083334dcdde24a3c56864114979b7a6f25a +R 1ee2743e0728dfd5b16085c25faca723 U dan -Z 8189e23423b7ab0255c97e908b40cc3e +Z 1978b788ff30c8d67f82270e25799f6e diff --git a/manifest.uuid b/manifest.uuid index cbb9ac7b02..c9913b975d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecab2083334dcdde24a3c56864114979b7a6f25a \ No newline at end of file +dcb8fa0f77a44250df0e8c4f6cfb9f6e181982d7 \ No newline at end of file diff --git a/test/fts4merge3.test b/test/fts4merge3.test index 30ea92cc8d..8cd3cf250c 100644 --- a/test/fts4merge3.test +++ b/test/fts4merge3.test @@ -88,7 +88,7 @@ do_all_bc_test { do_test 1.15 { sql2 { SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 - } } {5 1} + } } {6 1} } } diff --git a/test/trace2.test b/test/trace2.test index b1a53e77b5..06bb01c1fc 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -130,7 +130,7 @@ ifcapable fts3 { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" - "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" + "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" From dbd4f5cfb6886d59981f8047a6185e8d01e59763 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 11:09:59 +0000 Subject: [PATCH 24/68] When an incremental blob cursor is invalidated (occurs when an SQL statement modifies or deletes the row the blob cursor points to) release all page references held by the cursor. Otherwise, the presence of these references may cause other code in btree.c to incorrectly infer that the database is corrupt. FossilOrigin-Name: 82c3f2ba42f2c75ba6951cc2743148886a4dc0bc --- manifest | 13 ++++--- manifest.uuid | 2 +- src/btree.c | 38 ++++++++++--------- test/incrblob4.test | 90 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 test/incrblob4.test diff --git a/manifest b/manifest index 508b8682ec..42af1dead0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\sexisting\stest\scases. -D 2012-03-23T11:07:22.067 +C When\san\sincremental\sblob\scursor\sis\sinvalidated\s(occurs\swhen\san\sSQL\sstatement\smodifies\sor\sdeletes\sthe\srow\sthe\sblob\scursor\spoints\sto)\srelease\sall\spage\sreferences\sheld\sby\sthe\scursor.\sOtherwise,\sthe\spresence\sof\sthese\sreferences\smay\scause\sother\scode\sin\sbtree.c\sto\sincorrectly\sinfer\sthat\sthe\sdatabase\sis\scorrupt. +D 2012-03-23T11:09:59.261 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 6be23a344d3301ae38e92fddb3a33b91c309fce4 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 253c3147a4ebbaee42cd329dbdc0856200bbbda7 +F src/btree.c 02aeee1f6d425e11f7b9b2d9d461ac501645ed6f F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 @@ -520,6 +520,7 @@ F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/incrblob.test 26fde912a1e0aff158b3a84ef3b265f046aad3be F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 +F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 F test/incrblobfault.test 917c0292224c64a56ef7215fd633a3a82f805be0 F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 @@ -996,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P ecab2083334dcdde24a3c56864114979b7a6f25a -R 1ee2743e0728dfd5b16085c25faca723 +P dcb8fa0f77a44250df0e8c4f6cfb9f6e181982d7 +R 290f1c272852728c6b51f5c4cfcf23c3 U dan -Z 1978b788ff30c8d67f82270e25799f6e +Z a40c41b170a152c43db8b0f47b2a047b diff --git a/manifest.uuid b/manifest.uuid index c9913b975d..4d4b9fb32f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcb8fa0f77a44250df0e8c4f6cfb9f6e181982d7 \ No newline at end of file +82c3f2ba42f2c75ba6951cc2743148886a4dc0bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 68345144a8..d24c2be069 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6789,13 +6789,6 @@ int sqlite3BtreeInsert( ** blob of associated data. */ assert( (pKey==0)==(pCur->pKeyInfo==0) ); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); - } - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -6809,6 +6802,14 @@ int sqlite3BtreeInsert( */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); + } + if( !loc ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; @@ -6919,12 +6920,6 @@ int sqlite3BtreeDelete(BtCursor *pCur){ return SQLITE_ERROR; /* Something has gone awry. */ } - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; @@ -6950,6 +6945,13 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell); @@ -7231,13 +7233,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); - rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); diff --git a/test/incrblob4.test b/test/incrblob4.test new file mode 100644 index 0000000000..a96356b06f --- /dev/null +++ b/test/incrblob4.test @@ -0,0 +1,90 @@ +# 2012 March 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable {!incrblob} { finish_test ; return } +set testprefix incrblob4 + +proc create_t1 {} { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(k INTEGER PRIMARY KEY, v); + } +} + +proc populate_t1 {} { + set data [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + foreach d $data { + set blob [string repeat $d 900] + execsql { INSERT INTO t1(v) VALUES($blob) } + } +} + + +do_test 1.1 { + create_t1 + populate_t1 +} {} + +do_test 1.2 { + set blob [db incrblob t1 v 5] + read $blob 10 +} {eeeeeeeeee} + +do_test 1.3 { + execsql { DELETE FROM t1 } + populate_t1 +} {} + + + +do_test 2.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 2.2 { + set blob [db incrblob t1 v 10] + read $blob 10 +} {jjjjjjjjjj} + +do_test 2.3 { + set new [string repeat % 900] + execsql { DELETE FROM t1 WHERE k=10 } + execsql { DELETE FROM t1 WHERE k=9 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + + + +do_test 3.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 3.2 { + set blob [db incrblob t1 v 20] + read $blob 10 +} {tttttttttt} + +do_test 3.3 { + set new [string repeat % 900] + execsql { UPDATE t1 SET v = $new WHERE k = 20 } + execsql { DELETE FROM t1 WHERE k=19 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + +finish_test + From 865e26afa2e19f917c8c011aa37adfee0d269165 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 13:40:59 +0000 Subject: [PATCH 25/68] Fix another test case issue in trace2.test. FossilOrigin-Name: 02a8e4236cf2c1c74b3c6537be1b5f197175a32e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/trace2.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 42af1dead0..0c849c6500 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sincremental\sblob\scursor\sis\sinvalidated\s(occurs\swhen\san\sSQL\sstatement\smodifies\sor\sdeletes\sthe\srow\sthe\sblob\scursor\spoints\sto)\srelease\sall\spage\sreferences\sheld\sby\sthe\scursor.\sOtherwise,\sthe\spresence\sof\sthese\sreferences\smay\scause\sother\scode\sin\sbtree.c\sto\sincorrectly\sinfer\sthat\sthe\sdatabase\sis\scorrupt. -D 2012-03-23T11:09:59.261 +C Fix\sanother\stest\scase\sissue\sin\strace2.test. +D 2012-03-23T13:40:59.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -860,7 +860,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 9e4c0b3d4a85494acf1c1e077276d1e934100ce4 +F test/trace2.test c1dc104a8d11a347c870cfea6235e3fc6f6cb06d F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P dcb8fa0f77a44250df0e8c4f6cfb9f6e181982d7 -R 290f1c272852728c6b51f5c4cfcf23c3 +P 82c3f2ba42f2c75ba6951cc2743148886a4dc0bc +R 8a1a17690cd4174645797512470efb7b U dan -Z a40c41b170a152c43db8b0f47b2a047b +Z 4882f51212a5dac4b2ac7a10dc446a83 diff --git a/manifest.uuid b/manifest.uuid index 4d4b9fb32f..c786337c25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -82c3f2ba42f2c75ba6951cc2743148886a4dc0bc \ No newline at end of file +02a8e4236cf2c1c74b3c6537be1b5f197175a32e \ No newline at end of file diff --git a/test/trace2.test b/test/trace2.test index 06bb01c1fc..2f7ae7d5a2 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -131,7 +131,7 @@ ifcapable fts3 { "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" - "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" + "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" From d60f4f4c95c009816da2135a345c9eafba870bfc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 23 Mar 2012 14:23:52 +0000 Subject: [PATCH 26/68] When an incremental blob cursor is invalidated (occurs when an SQL statement modifies or deletes the row the blob cursor points to) release all page references held by the cursor. Otherwise, the presence of these references may cause other code in btree.c to incorrectly infer that the database is corrupt. FossilOrigin-Name: 341b703ce16361a64ed8bba64ff46792132c0b56 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/btree.c | 38 ++++++++++--------- test/incrblob4.test | 90 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 25 deletions(-) create mode 100644 test/incrblob4.test diff --git a/manifest b/manifest index a8b57dd4e9..2a0edd6018 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\squote\sthe\snames\sof\stables\sin\sthe\soutput\sof\sthe\sshell's\s".dump"\scommand,\neven\sif\sthe\sname\sis\spure\salphabetic\stext,\sin\scase\sthe\sname\sis\sa\skeyword. -D 2012-03-22T12:50:34.137 +C When\san\sincremental\sblob\scursor\sis\sinvalidated\s(occurs\swhen\san\sSQL\sstatement\smodifies\sor\sdeletes\sthe\srow\sthe\sblob\scursor\spoints\sto)\srelease\sall\spage\sreferences\sheld\sby\sthe\scursor.\sOtherwise,\sthe\spresence\sof\sthese\sreferences\smay\scause\sother\scode\sin\sbtree.c\sto\sincorrectly\sinfer\sthat\sthe\sdatabase\sis\scorrupt. +D 2012-03-23T14:23:52.727 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 6be23a344d3301ae38e92fddb3a33b91c309fce4 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 253c3147a4ebbaee42cd329dbdc0856200bbbda7 +F src/btree.c 02aeee1f6d425e11f7b9b2d9d461ac501645ed6f F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 @@ -516,6 +516,7 @@ F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/incrblob.test 26fde912a1e0aff158b3a84ef3b265f046aad3be F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 +F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 F test/incrblobfault.test 917c0292224c64a56ef7215fd633a3a82f805be0 F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 @@ -992,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 0fb26c7bfa7a4bb1503f90fd6f5b9c70f444665b -R 76825aab78a4aec4e0f8b3a0fc8144af +P 638b71150281a211f89b4057b0d5d32d3fbcf323 +R bf4ea56b709d8c66bd287b12bb865310 U drh -Z cbcc6857e5f2528670826efd0ad94a0c +Z 71976091f9202f9aa459dff852dbcf55 diff --git a/manifest.uuid b/manifest.uuid index f6c5dfb8e2..a8830b75fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -638b71150281a211f89b4057b0d5d32d3fbcf323 \ No newline at end of file +341b703ce16361a64ed8bba64ff46792132c0b56 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 68345144a8..d24c2be069 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6789,13 +6789,6 @@ int sqlite3BtreeInsert( ** blob of associated data. */ assert( (pKey==0)==(pCur->pKeyInfo==0) ); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); - } - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -6809,6 +6802,14 @@ int sqlite3BtreeInsert( */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); + } + if( !loc ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; @@ -6919,12 +6920,6 @@ int sqlite3BtreeDelete(BtCursor *pCur){ return SQLITE_ERROR; /* Something has gone awry. */ } - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; @@ -6950,6 +6945,13 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell); @@ -7231,13 +7233,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); - rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); diff --git a/test/incrblob4.test b/test/incrblob4.test new file mode 100644 index 0000000000..a96356b06f --- /dev/null +++ b/test/incrblob4.test @@ -0,0 +1,90 @@ +# 2012 March 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable {!incrblob} { finish_test ; return } +set testprefix incrblob4 + +proc create_t1 {} { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(k INTEGER PRIMARY KEY, v); + } +} + +proc populate_t1 {} { + set data [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + foreach d $data { + set blob [string repeat $d 900] + execsql { INSERT INTO t1(v) VALUES($blob) } + } +} + + +do_test 1.1 { + create_t1 + populate_t1 +} {} + +do_test 1.2 { + set blob [db incrblob t1 v 5] + read $blob 10 +} {eeeeeeeeee} + +do_test 1.3 { + execsql { DELETE FROM t1 } + populate_t1 +} {} + + + +do_test 2.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 2.2 { + set blob [db incrblob t1 v 10] + read $blob 10 +} {jjjjjjjjjj} + +do_test 2.3 { + set new [string repeat % 900] + execsql { DELETE FROM t1 WHERE k=10 } + execsql { DELETE FROM t1 WHERE k=9 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + + + +do_test 3.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 3.2 { + set blob [db incrblob t1 v 20] + read $blob 10 +} {tttttttttt} + +do_test 3.3 { + set new [string repeat % 900] + execsql { UPDATE t1 SET v = $new WHERE k = 20 } + execsql { DELETE FROM t1 WHERE k=19 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + +finish_test + From 3501a916774fcc3f071b36d284f52b5bb33bf942 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 14:38:49 +0000 Subject: [PATCH 27/68] Remove an incorrect assert() statement. Fix a const-related warning. FossilOrigin-Name: 96ed47493b3d46344fd2105642f31690aee06674 --- ext/fts3/fts3_write.c | 3 +-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts4merge.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 5b8fcc2ab2..e755c53cc6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3285,7 +3285,6 @@ static int fts3IncrmergeCsr( int nByte; /* Bytes allocated at pCsr->apSegment[] */ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ - assert( nSeg>=2 ); memset(pCsr, 0, sizeof(*pCsr)); nByte = sizeof(Fts3SegReader *) * nSeg; pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); @@ -4389,7 +4388,7 @@ static int fts3IncrmergeHintLoad( if( rc==SQLITE_OK ){ sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); if( SQLITE_ROW==sqlite3_step(pSelect) ){ - char *aHint = sqlite3_column_blob(pSelect, 0); + const char *aHint = sqlite3_column_blob(pSelect, 0); int nHint = sqlite3_column_bytes(pSelect, 0); if( aHint ){ int i; diff --git a/manifest b/manifest index 0c849c6500..23ef80f398 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\stest\scase\sissue\sin\strace2.test. -D 2012-03-23T13:40:59.728 +C Remove\san\sincorrect\sassert()\sstatement.\sFix\sa\sconst-related\swarning. +D 2012-03-23T14:38:49.144 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 62ada731e3beef4977386a91240c5f80a1db884e +F ext/fts3/fts3_write.c 8791a3dae9d1433261800e7d36208d3e88ffa5bc F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -498,7 +498,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 663607bef5575027295d0b7168370ccd0bae6522 +F test/fts4merge.test 7cea27240f68596bc1b02117409d34a3f7e8c3ec F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 82c3f2ba42f2c75ba6951cc2743148886a4dc0bc -R 8a1a17690cd4174645797512470efb7b +P 02a8e4236cf2c1c74b3c6537be1b5f197175a32e +R 83c8ccbccee5d73d8055d59ab542cc92 U dan -Z 4882f51212a5dac4b2ac7a10dc446a83 +Z 9064fda8da51ce4c65bcaaf3383b8b6d diff --git a/manifest.uuid b/manifest.uuid index c786337c25..e3a12a140a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02a8e4236cf2c1c74b3c6537be1b5f197175a32e \ No newline at end of file +96ed47493b3d46344fd2105642f31690aee06674 \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 5d3368e528..887888f899 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -162,6 +162,7 @@ do_execsql_test 4.4.2 { SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; } "0 {0 1 2 3 4 5} 1 0" + #------------------------------------------------------------------------- # Test cases 5.* # @@ -255,5 +256,35 @@ do_execsql_test 5.11 { X'0000' } +#------------------------------------------------------------------------- +# Test cases 6.* +# +# At one point the following test caused an assert() to fail (because the +# second 'merge=1,2' operation below actually "merges" a single input +# segment, which was unexpected). +# +do_test 6.1 { + reset_db + set a [string repeat a 900] + set b [string repeat b 900] + set c [string repeat c 900] + set d [string repeat d 900] + execsql { + CREATE VIRTUAL TABLE t1 USING fts4; + BEGIN; + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + COMMIT; + } + + execsql { + INSERT INTO t1(t1) VALUES('merge=1,2'); + INSERT INTO t1(t1) VALUES('merge=1,2'); + } +} {} finish_test From ba512b0b3d10e4b5b080f81fce2796662a41c207 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 15:38:43 +0000 Subject: [PATCH 28/68] Add a test to verify that sqlite3_total_changes() works with incr-merge operations. FossilOrigin-Name: 1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts4merge.test | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 23ef80f398..aa8e07fecd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sincorrect\sassert()\sstatement.\sFix\sa\sconst-related\swarning. -D 2012-03-23T14:38:49.144 +C Add\sa\stest\sto\sverify\sthat\ssqlite3_total_changes()\sworks\swith\sincr-merge\soperations. +D 2012-03-23T15:38:43.298 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -498,7 +498,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 7cea27240f68596bc1b02117409d34a3f7e8c3ec +F test/fts4merge.test 120e0baf17a01f0cb696d6f6f9b6506e1587ef90 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 02a8e4236cf2c1c74b3c6537be1b5f197175a32e -R 83c8ccbccee5d73d8055d59ab542cc92 +P 96ed47493b3d46344fd2105642f31690aee06674 +R f6fdda04671e81d1436014dc3e6ce1ef U dan -Z 9064fda8da51ce4c65bcaaf3383b8b6d +Z ca34b2031de3681fa84c177cb5bd2b1e diff --git a/manifest.uuid b/manifest.uuid index e3a12a140a..2ffc66650d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96ed47493b3d46344fd2105642f31690aee06674 \ No newline at end of file +1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 887888f899..422d585cbe 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -287,4 +287,44 @@ do_test 6.1 { } } {} +#------------------------------------------------------------------------- +# Test cases 7.* +# +# Test that the value returned by sqlite3_total_changes() increases by +# 1 following a no-op "merge=A,B", or by more than 1 if actual work is +# performed. +# +do_test 7.0 { + reset_db + fts3_build_db_1 1000 +} {} + +do_execsql_test 7.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} +do_test 7.2 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } + expr { ([db total_changes] - $x)>1 } +} {1} +do_test 7.3 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {1} +do_test 7.4 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {0} +do_test 7.5 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {0} + finish_test From d59de73e27528c79ed63bf42993512a2a1a60d55 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Mar 2012 18:26:11 +0000 Subject: [PATCH 29/68] Fix a spurious SQLITE_CONSTRAINT error that may be returned by an incr-merge operation. FossilOrigin-Name: ed7c17ea165f6348506bd23ebc58c427bb65d697 --- ext/fts3/fts3.c | 6 ++- ext/fts3/fts3Int.h | 3 +- ext/fts3/fts3_write.c | 87 ++++++++++++++++++++++++++++++++++++------- manifest | 16 ++++---- manifest.uuid | 2 +- 5 files changed, 89 insertions(+), 25 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 421052b937..ad0cb36da9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3412,11 +3412,15 @@ static int fts3RenameMethod( ** Flush the contents of the pending-terms table to disk. */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - return fts3SyncMethod(pVtab); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; } /* diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index e847aad15d..f84d15d196 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -200,7 +200,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[35]; + sqlite3_stmt *aStmt[36]; char *zReadExprlist; char *zWriteExprlist; @@ -209,6 +209,7 @@ struct Fts3Table { u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index e755c53cc6..30f5e666eb 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -259,10 +259,11 @@ struct SegmentNode { #define SQL_FIND_MERGE_LEVEL 28 #define SQL_MAX_LEAF_NODE_ESTIMATE 29 #define SQL_DELETE_SEGDIR_ENTRY 30 -#define SQL_SHIFT_SEGDIR_ENTRIES 31 +#define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 /* ** This function is used to obtain an SQLite prepared statement handle @@ -337,10 +338,10 @@ static int fts3SqlStmt( ** Delete the %_segdir entry on absolute level :1 with index :2. */ /* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", -/* SQL_SHIFT_SEGDIR_ENTRIES -** Reduce by one the idx values of all segments on absolute level :1 with -** an index greater than :2. */ -/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = idx - 1 WHERE level = ? AND idx>:2", +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", /* SQL_SELECT_SEGDIR ** Read a single entry from the %_segdir table. The entry from absolute @@ -357,7 +358,11 @@ static int fts3SqlStmt( /* SQL_SEGMENT_IS_APPENDABLE ** Return a single row if the segment with end_block=? is appendable. Or ** no rows otherwise. */ -/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL" +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -4113,7 +4118,6 @@ static int fts3RemoveSegdirEntry( ){ int rc; /* Return code */ sqlite3_stmt *pDelete = 0; /* DELETE statement */ - sqlite3_stmt *pUpdate = 0; /* UPDATE statement */ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); if( rc==SQLITE_OK ){ @@ -4123,16 +4127,67 @@ static int fts3RemoveSegdirEntry( rc = sqlite3_reset(pDelete); } + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRIES, &pUpdate, 0); - } - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pUpdate, 1, iAbsLevel); - sqlite3_bind_int(pUpdate, 2, iIdx); - sqlite3_step(pUpdate); - rc = sqlite3_reset(pUpdate); + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); return rc; } @@ -4335,6 +4390,10 @@ static int fts3IncrmergeChomp( } } + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + *pnRem = nRem; return rc; } diff --git a/manifest b/manifest index aa8e07fecd..a6dafc26fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sto\sverify\sthat\ssqlite3_total_changes()\sworks\swith\sincr-merge\soperations. -D 2012-03-23T15:38:43.298 +C Fix\sa\sspurious\sSQLITE_CONSTRAINT\serror\sthat\smay\sbe\sreturned\sby\san\sincr-merge\soperation. +D 2012-03-23T18:26:11.608 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c a62e09140c00f9c401efca661e7f2ae9909194f6 +F ext/fts3/fts3.c c47bb6fe7a6bde82cbdf4c504302221710699d64 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h caa745f80405bc0c0a45a2f83e120d5a2c13753c +F ext/fts3/fts3Int.h 5fe1651db88206ca68f421cf62bb8501395677a3 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 8791a3dae9d1433261800e7d36208d3e88ffa5bc +F ext/fts3/fts3_write.c e3b750530132b3237aed3e5f008845cc4d7865f9 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 96ed47493b3d46344fd2105642f31690aee06674 -R f6fdda04671e81d1436014dc3e6ce1ef +P 1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 +R c0998f3b8bc74543bb7322ba65e1257e U dan -Z ca34b2031de3681fa84c177cb5bd2b1e +Z 3e48c8e93d6fd57a390b8a97532e70ea diff --git a/manifest.uuid b/manifest.uuid index 2ffc66650d..0886bc8716 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 \ No newline at end of file +ed7c17ea165f6348506bd23ebc58c427bb65d697 \ No newline at end of file From 4ef9dfff97802aa3eec9fed2f7cbfe833ff2d04b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 24 Mar 2012 02:20:43 +0000 Subject: [PATCH 30/68] An attempt at automatic incremental merging for FTS4. FossilOrigin-Name: ed69434cd89084f4b57bd2cc4f5cc558904af565 --- ext/fts3/fts3.c | 12 +++++- ext/fts3/fts3Int.h | 5 +++ ext/fts3/fts3_write.c | 85 +++++++++++++++++++++++++++++++++++++------ manifest | 23 +++++++----- manifest.uuid | 2 +- test/bc_common.tcl | 6 +-- 6 files changed, 104 insertions(+), 29 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index ad0cb36da9..6a68d45e45 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1276,6 +1276,7 @@ static int fts3InitVtab( p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -3102,8 +3103,14 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>0 ){ + int A = p->nLeafAdd * p->mxLevel; + A += A/2; + rc = sqlite3Fts3Incrmerge(p, A, 8); + } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -3118,6 +3125,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index f84d15d196..ddf3a6b133 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -196,6 +196,9 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ + u8 bAutoincrmerge; /* True if automerge=1 */ + int mxLevel; /* Maximum level seen on this transaction */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. @@ -479,6 +482,8 @@ struct Fts3MultiSegReader { int nDoclist; /* Size of aDoclist[] in bytes */ }; +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 30f5e666eb..e9fe87c492 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -72,6 +72,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; */ #define FTS_STAT_DOCTOTAL 0 #define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic @@ -330,7 +331,7 @@ static int fts3SqlStmt( /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See -** function fts3Incrmerge() for details. */ +** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", @@ -1868,6 +1869,14 @@ static int fts3WriteSegment( return rc; } +/* +** Update the Fts3Table.mxLevel field, if appropriate +*/ +static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ + iLevel %= FTS3_SEGDIR_MAXLEVEL; + if( iLevel>p->mxLevel ) p->mxLevel = iLevel; +} + /* ** Insert a record into the %_segdir table. */ @@ -1885,6 +1894,7 @@ static int fts3WriteSegdir( int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); + fts3UpdateMaxLevel(p, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); @@ -2369,6 +2379,7 @@ static int fts3SegmentMaxLevel( ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); + fts3UpdateMaxLevel(p, *pnMax); } return sqlite3_reset(pStmt); } @@ -2999,11 +3010,29 @@ static int fts3SegmentMerge( int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + + p->nLeafAdd += (p->nPendingData + p->nNodeSize - 1)/p->nNodeSize; for(i=0; rc==SQLITE_OK && inIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + rc = sqlite3_reset(pStmt); + } + } return rc; } @@ -4472,7 +4501,7 @@ static int fts3IncrmergeHintLoad( ** nMin segments. Multiple merges might occur in an attempt to write the ** quota of nMerge leaf blocks. */ -static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ int bUseHint = 1; /* True if hint has not yet been attempted */ @@ -4578,6 +4607,19 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ return rc; } +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + /* ** Process statements of the form: ** @@ -4597,30 +4639,49 @@ static int fts3DoIncrmerge( const char *z = zParam; /* Read the first integer value */ - for(z=zParam; z[0]>='0' && z[0]<='9'; z++){ - nMerge = nMerge * 10 + (z[0] - '0'); - } + nMerge = fts3Getint(&z); /* If the first integer value is followed by a ',', read the second ** integer value. */ if( z[0]==',' && z[1]!='\0' ){ z++; - nMin = 0; - while( z[0]>='0' && z[0]<='9' ){ - nMin = nMin * 10 + (z[0] - '0'); - z++; - } + nMin = fts3Getint(&z); } if( z[0]!='\0' || nMin<2 ){ rc = SQLITE_ERROR; }else{ - rc = fts3Incrmerge(p, nMerge, nMin); + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); sqlite3Fts3SegmentsClose(p); } return rc; } +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + p->bAutoincrmerge = fts3Getint(&zParam)!=0; + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc;; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + + /* ** Handle a 'special' INSERT of the form: ** @@ -4642,6 +4703,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoRebuild(p); }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); diff --git a/manifest b/manifest index a6dafc26fe..b62cac476e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sspurious\sSQLITE_CONSTRAINT\serror\sthat\smay\sbe\sreturned\sby\san\sincr-merge\soperation. -D 2012-03-23T18:26:11.608 +C An\sattempt\sat\sautomatic\sincremental\smerging\sfor\sFTS4. +D 2012-03-24T02:20:43.242 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c c47bb6fe7a6bde82cbdf4c504302221710699d64 +F ext/fts3/fts3.c ef3d7af4a635beacf5546ea6c8c03329e263fd30 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 5fe1651db88206ca68f421cf62bb8501395677a3 +F ext/fts3/fts3Int.h 1e635c9ba3a384496a4524f7b94d7d0525930552 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c e3b750530132b3237aed3e5f008845cc4d7865f9 +F ext/fts3/fts3_write.c 663f2fa51e7096d8f496f94db5f215c6d8d22820 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -295,7 +295,7 @@ F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f -F test/bc_common.tcl df4c51ae0de0cb414a18027be2abb2bd7693ce7a +F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070 F test/bigfile.test 82dfe93ee7eb9e2e05641afa2b39ffd947a92ff1 F test/bigfile2.test 852f948cb492aadab45b58f4d2f3b0832a115cb0 @@ -997,7 +997,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1c72cecc6bf5be2a5c04ad6214a6bac22a29f860 -R c0998f3b8bc74543bb7322ba65e1257e -U dan -Z 3e48c8e93d6fd57a390b8a97532e70ea +P ed7c17ea165f6348506bd23ebc58c427bb65d697 +R c65796e564760bf2b3e7eed574e1eb1f +T *branch * fts4-auto-incr-merge +T *sym-fts4-auto-incr-merge * +T -sym-fts4-incr-merge * +U drh +Z 9b979efabd6b6c201cc4798b465233ce diff --git a/manifest.uuid b/manifest.uuid index 0886bc8716..4f3a06edaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed7c17ea165f6348506bd23ebc58c427bb65d697 \ No newline at end of file +ed69434cd89084f4b57bd2cc4f5cc558904af565 \ No newline at end of file diff --git a/test/bc_common.tcl b/test/bc_common.tcl index 8bba0245a5..eb9b6db9d3 100644 --- a/test/bc_common.tcl +++ b/test/bc_common.tcl @@ -17,7 +17,7 @@ proc bc_find_binaries {zCaption} { if {[llength $binaries]==0} { puts "WARNING: No historical binaries to test against." - puts "WARNING: Omitting backwards-compatibility tests $zFile" + puts "WARNING: Omitting backwards-compatibility tests" } foreach bin $binaries { @@ -70,7 +70,3 @@ proc do_all_bc_test {script} { uplevel [list do_bc_test $bin $script] } } - - - - From 4b1e4dabc31891cd8ebe458df992c1ef15345652 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Mar 2012 14:45:19 +0000 Subject: [PATCH 31/68] Modify the way the number of leaves written and the maximum relative level are calculated in the auto-incr-merge code. FossilOrigin-Name: 0d841c957c6ec4afecb49504177c6279e09d7012 --- ext/fts3/fts3.c | 15 +++++++++++---- ext/fts3/fts3Int.h | 3 ++- ext/fts3/fts3_write.c | 31 +++++++++++++++++++++++++++++-- manifest | 21 +++++++++------------ manifest.uuid | 2 +- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6a68d45e45..e8b38f2ed3 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3103,12 +3103,19 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ + const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>0 ){ - int A = p->nLeafAdd * p->mxLevel; - A += A/2; - rc = sqlite3Fts3Incrmerge(p, A, 8); + + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * p->mxLevel; + A += (A/2); + if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); } sqlite3Fts3SegmentsClose(p); return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index ddf3a6b133..abf5b76c97 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -203,7 +203,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[36]; + sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; @@ -431,6 +431,7 @@ int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index e9fe87c492..51f47d0106 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -265,6 +265,7 @@ struct SegmentNode { #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle @@ -363,7 +364,11 @@ static int fts3SqlStmt( /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ -/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC" +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -1877,6 +1882,27 @@ static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ if( iLevel>p->mxLevel ) p->mxLevel = iLevel; } +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + /* ** Insert a record into the %_segdir table. */ @@ -2194,6 +2220,7 @@ static int fts3SegWriterAdd( /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: @@ -2302,6 +2329,7 @@ static int fts3SegWriterFlush( rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } @@ -3011,7 +3039,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; - p->nLeafAdd += (p->nPendingData + p->nNodeSize - 1)/p->nNodeSize; for(i=0; rc==SQLITE_OK && inIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; diff --git a/manifest b/manifest index b62cac476e..255b73d910 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\sattempt\sat\sautomatic\sincremental\smerging\sfor\sFTS4. -D 2012-03-24T02:20:43.242 +C Modify\sthe\sway\sthe\snumber\sof\sleaves\swritten\sand\sthe\smaximum\srelative\slevel\sare\scalculated\sin\sthe\sauto-incr-merge\scode. +D 2012-03-24T14:45:19.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ef3d7af4a635beacf5546ea6c8c03329e263fd30 +F ext/fts3/fts3.c 442f0ccf5c1a6a63844b9712b6f66bf8adb603b6 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 1e635c9ba3a384496a4524f7b94d7d0525930552 +F ext/fts3/fts3Int.h f65df0fd76617cd43cb9795e5eec27e90ee4cbab F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 663f2fa51e7096d8f496f94db5f215c6d8d22820 +F ext/fts3/fts3_write.c c71f84711f189bc18a82560949ac1656318fd2c5 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -997,10 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P ed7c17ea165f6348506bd23ebc58c427bb65d697 -R c65796e564760bf2b3e7eed574e1eb1f -T *branch * fts4-auto-incr-merge -T *sym-fts4-auto-incr-merge * -T -sym-fts4-incr-merge * -U drh -Z 9b979efabd6b6c201cc4798b465233ce +P ed69434cd89084f4b57bd2cc4f5cc558904af565 +R bf494068b52a47b15eebd4119e5e6c0c +U dan +Z 7eb3af40022cc7568ef4099b19e62140 diff --git a/manifest.uuid b/manifest.uuid index 4f3a06edaf..6d50369d05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed69434cd89084f4b57bd2cc4f5cc558904af565 \ No newline at end of file +0d841c957c6ec4afecb49504177c6279e09d7012 \ No newline at end of file From 45eddd685577924dd4e26b8b42e9efe68f70c7d4 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Mar 2012 16:11:21 +0000 Subject: [PATCH 32/68] Remove the Fts3Table.mxLevel variable. FossilOrigin-Name: 67a0cffc9d07be7f09dad9d019a18160711295cd --- ext/fts3/fts3.c | 2 +- ext/fts3/fts3Int.h | 1 - ext/fts3/fts3_write.c | 10 ---------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 10 insertions(+), 21 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e8b38f2ed3..f874dfa7f3 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3113,7 +3113,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ rc = sqlite3Fts3MaxLevel(p, &mxLevel); assert( rc==SQLITE_OK || mxLevel==0 ); - A = p->nLeafAdd * p->mxLevel; + A = p->nLeafAdd * mxLevel; A += (A/2); if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index abf5b76c97..af310c5738 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -197,7 +197,6 @@ struct Fts3Table { char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ u8 bAutoincrmerge; /* True if automerge=1 */ - int mxLevel; /* Maximum level seen on this transaction */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 51f47d0106..8919962baf 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1874,14 +1874,6 @@ static int fts3WriteSegment( return rc; } -/* -** Update the Fts3Table.mxLevel field, if appropriate -*/ -static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ - iLevel %= FTS3_SEGDIR_MAXLEVEL; - if( iLevel>p->mxLevel ) p->mxLevel = iLevel; -} - /* ** Find the largest relative level number in the table. If successful, set ** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, @@ -1920,7 +1912,6 @@ static int fts3WriteSegdir( int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); - fts3UpdateMaxLevel(p, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); @@ -2407,7 +2398,6 @@ static int fts3SegmentMaxLevel( ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); - fts3UpdateMaxLevel(p, *pnMax); } return sqlite3_reset(pStmt); } diff --git a/manifest b/manifest index fbaa006611..ae8fb47763 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sauto-incr-merge\swith\sincr-merge\sbranch. -D 2012-03-24T14:45:59.366 +C Remove\sthe\sFts3Table.mxLevel\svariable. +D 2012-03-24T16:11:21.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 442f0ccf5c1a6a63844b9712b6f66bf8adb603b6 +F ext/fts3/fts3.c eb3a57bb7913e83727f6bcb1142375df5637478f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h f65df0fd76617cd43cb9795e5eec27e90ee4cbab +F ext/fts3/fts3Int.h 6d4ffaca18df57533a7d6240dbdd835c4f3f096a F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c c71f84711f189bc18a82560949ac1656318fd2c5 +F ext/fts3/fts3_write.c 9c9d3062f50d35a0f8b88e5e8f926b622cc210aa F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P ed7c17ea165f6348506bd23ebc58c427bb65d697 0d841c957c6ec4afecb49504177c6279e09d7012 -R bf494068b52a47b15eebd4119e5e6c0c +P 1c68687ab6d05b100191663820e7d82377d52445 +R f7873f1fffcc305e18231176cdb210c4 U dan -Z a37d5b74ba1d4a69b01095005645cefa +Z c98b0e8c409bca973107326748221c8a diff --git a/manifest.uuid b/manifest.uuid index 6bd6c7d918..c164bc0827 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c68687ab6d05b100191663820e7d82377d52445 \ No newline at end of file +67a0cffc9d07be7f09dad9d019a18160711295cd \ No newline at end of file From 790964ebc45da08d4de8dfa25ebfb4223e89c2c4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 24 Mar 2012 16:18:08 +0000 Subject: [PATCH 33/68] Fix a bug in debugging code enabled when FTS3_LOG_MERGES is defined. FossilOrigin-Name: 2e06babf49de844d0e878d68114dbcc5ad4d6e54 --- ext/fts3/fts3_write.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 8919962baf..a92081f871 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1127,7 +1127,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - fts3LogMerge(16, getAbsoluteLevel(iLevel, iLangid, iIndex)); + fts3LogMerge(16, getAbsoluteLevel(p, iLevel, iLangid, iIndex)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ diff --git a/manifest b/manifest index ae8fb47763..b3e124c856 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sFts3Table.mxLevel\svariable. -D 2012-03-24T16:11:21.049 +C Fix\sa\sbug\sin\sdebugging\scode\senabled\swhen\sFTS3_LOG_MERGES\sis\sdefined. +D 2012-03-24T16:18:08.837 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 9c9d3062f50d35a0f8b88e5e8f926b622cc210aa +F ext/fts3/fts3_write.c d96bd53e22074c4b35b3db3b9544ac3e55616333 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1c68687ab6d05b100191663820e7d82377d52445 -R f7873f1fffcc305e18231176cdb210c4 -U dan -Z c98b0e8c409bca973107326748221c8a +P 67a0cffc9d07be7f09dad9d019a18160711295cd +R b9a75c391c5508df678e65c2562d628a +U drh +Z 4e5e71ac5e6024845db251d7d739e1ab diff --git a/manifest.uuid b/manifest.uuid index c164bc0827..9efe1a35d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67a0cffc9d07be7f09dad9d019a18160711295cd \ No newline at end of file +2e06babf49de844d0e878d68114dbcc5ad4d6e54 \ No newline at end of file From 0dfcdeb4f0b59fc06c8562385fdd161b6e3a04a9 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Mar 2012 16:43:55 +0000 Subject: [PATCH 34/68] Add a comment to fts3SyncMethod() to justify the nMinMerge=64 constant. FossilOrigin-Name: af55ca5fc6778cb6d1a79a17dfa2d4e567ea1ccc --- ext/fts3/fts3.c | 21 +++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f874dfa7f3..cf9fe64ed9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3103,7 +3103,28 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); diff --git a/manifest b/manifest index b3e124c856..b81062a1dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sdebugging\scode\senabled\swhen\sFTS3_LOG_MERGES\sis\sdefined. -D 2012-03-24T16:18:08.837 +C Add\sa\scomment\sto\sfts3SyncMethod()\sto\sjustify\sthe\snMinMerge=64\sconstant. +D 2012-03-24T16:43:55.163 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c eb3a57bb7913e83727f6bcb1142375df5637478f +F ext/fts3/fts3.c 10988e19ba78d846e5979a5118843e22822a0147 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 6d4ffaca18df57533a7d6240dbdd835c4f3f096a F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 67a0cffc9d07be7f09dad9d019a18160711295cd -R b9a75c391c5508df678e65c2562d628a -U drh -Z 4e5e71ac5e6024845db251d7d739e1ab +P 2e06babf49de844d0e878d68114dbcc5ad4d6e54 +R 432c0c36d101ff48a9d07485fc32e9b8 +U dan +Z 314cc4329e06c95183ea81ebec52b3f5 diff --git a/manifest.uuid b/manifest.uuid index 9efe1a35d6..9b8d8efdbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e06babf49de844d0e878d68114dbcc5ad4d6e54 \ No newline at end of file +af55ca5fc6778cb6d1a79a17dfa2d4e567ea1ccc \ No newline at end of file From 61fa09dbf2ecd85541aa9788809ba592b8c635c7 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Mar 2012 17:09:11 +0000 Subject: [PATCH 35/68] Fix a failing assert() in the FTS3_LOG_MERGES related code. FossilOrigin-Name: 4220d52cb3426f1680b72d57ecc9f4ade029357d --- ext/fts3/fts3_write.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index a92081f871..ad4e5e4887 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1127,7 +1127,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - fts3LogMerge(16, getAbsoluteLevel(p, iLevel, iLangid, iIndex)); + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ diff --git a/manifest b/manifest index b81062a1dd..252baaa8e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sfts3SyncMethod()\sto\sjustify\sthe\snMinMerge=64\sconstant. -D 2012-03-24T16:43:55.163 +C Fix\sa\sfailing\sassert()\sin\sthe\sFTS3_LOG_MERGES\srelated\scode. +D 2012-03-24T17:09:11.161 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c d96bd53e22074c4b35b3db3b9544ac3e55616333 +F ext/fts3/fts3_write.c ee3aeaa51aa633ada26e6d14073b1f893f094973 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -997,7 +997,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 2e06babf49de844d0e878d68114dbcc5ad4d6e54 -R 432c0c36d101ff48a9d07485fc32e9b8 +P af55ca5fc6778cb6d1a79a17dfa2d4e567ea1ccc +R c679b33a6e75fcdd2f180faacc012668 U dan -Z 314cc4329e06c95183ea81ebec52b3f5 +Z 4d0762459d11aa2e9f32a4956c0ceb5b diff --git a/manifest.uuid b/manifest.uuid index 9b8d8efdbf..5d8bce9828 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af55ca5fc6778cb6d1a79a17dfa2d4e567ea1ccc \ No newline at end of file +4220d52cb3426f1680b72d57ecc9f4ade029357d \ No newline at end of file From 6c2e7e19fa7f03707ac80961d5c29d3ba5095ea7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 24 Mar 2012 17:29:05 +0000 Subject: [PATCH 36/68] Enable fts3 tables to use incremental merge by automatically creating the %_stat table when it is needed. FossilOrigin-Name: cc051fc0b2d89603b27b94cf2afdbda417ee9d94 --- ext/fts3/fts3.c | 27 +++- ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_write.c | 12 +- manifest | 19 +-- manifest.uuid | 2 +- test/fts3merge.test | 330 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 375 insertions(+), 16 deletions(-) create mode 100644 test/fts3merge.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index cf9fe64ed9..63661393a5 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -570,6 +570,18 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ } } +/* +** Create the %_stat table if it does not already exist. +*/ +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done @@ -631,10 +643,7 @@ static int fts3CreateTables(Fts3Table *p){ ); } if( p->bHasStat ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); + sqlite3Fts3CreateStatTable(&rc, p); } return rc; } @@ -1329,6 +1338,16 @@ static int fts3InitVtab( rc = fts3CreateTables(p); } + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + int rc2 = SQLITE_OK; + fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", + p->zDb, p->zName); + if( rc2==SQLITE_OK ) p->bHasStat = 1; + } + /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index af310c5738..0ce67a2d10 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -493,6 +493,7 @@ void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +void sqlite3Fts3CreateStatTable(int*, Fts3Table*); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ad4e5e4887..2d3a91d87e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4668,7 +4668,11 @@ static int fts3DoIncrmerge( if( z[0]!='\0' || nMin<2 ){ rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + rc = SQLITE_OK; + if( !p->bHasStat ) sqlite3Fts3CreateStatTable(&rc, p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } sqlite3Fts3SegmentsClose(p); } return rc; @@ -4686,9 +4690,13 @@ static int fts3DoAutoincrmerge( Fts3Table *p, /* FTS3 table handle */ const char *zParam /* Nul-terminated string containing boolean */ ){ - int rc; + int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; p->bAutoincrmerge = fts3Getint(&zParam)!=0; + if( !p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc;; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); diff --git a/manifest b/manifest index 252baaa8e3..3c55a976b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfailing\sassert()\sin\sthe\sFTS3_LOG_MERGES\srelated\scode. -D 2012-03-24T17:09:11.161 +C Enable\sfts3\stables\sto\suse\sincremental\smerge\sby\sautomatically\screating\sthe\n%_stat\stable\swhen\sit\sis\sneeded. +D 2012-03-24T17:29:05.827 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 10988e19ba78d846e5979a5118843e22822a0147 +F ext/fts3/fts3.c 95409b49801ee7736755d7e307e606571b754a58 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 6d4ffaca18df57533a7d6240dbdd835c4f3f096a +F ext/fts3/fts3Int.h eb749124db7c94b6f89d793cdd4d993a52c46646 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c ee3aeaa51aa633ada26e6d14073b1f893f094973 +F ext/fts3/fts3_write.c 6a092ee27198716969bfbaa2194aa67eabeb2ff6 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -487,6 +487,7 @@ F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 +F test/fts3merge.test acb0be43658029565e7b448f8968149de80549d7 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5 @@ -997,7 +998,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P af55ca5fc6778cb6d1a79a17dfa2d4e567ea1ccc -R c679b33a6e75fcdd2f180faacc012668 -U dan -Z 4d0762459d11aa2e9f32a4956c0ceb5b +P 4220d52cb3426f1680b72d57ecc9f4ade029357d +R d86044022236a45ab8294cd3002625f6 +U drh +Z 5e5822ed626477e734b96980ea604331 diff --git a/manifest.uuid b/manifest.uuid index 5d8bce9828..1019a1397f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4220d52cb3426f1680b72d57ecc9f4ade029357d \ No newline at end of file +cc051fc0b2d89603b27b94cf2afdbda417ee9d94 \ No newline at end of file diff --git a/test/fts3merge.test b/test/fts3merge.test new file mode 100644 index 0000000000..a0854fa1a4 --- /dev/null +++ b/test/fts3merge.test @@ -0,0 +1,330 @@ +# 2012 March 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the incremental merge function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts3merge + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test cases 1.* +# +do_test 1.0 { fts3_build_db_1 1004 } {} +do_test 1.1 { fts3_integrity_check t1 } {ok} +do_execsql_test 1.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} + +for {set i 0} {$i<20} {incr i} { + do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } + do_test 1.2.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.2.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} +} + +do_execsql_test 1.3 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3} + 1 {0 1 2 3 4 5 6} + 2 {0 1 2 3} +} + +for {set i 0} {$i<100} {incr i} { + do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } + do_test 1.4.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.4.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} +} + +do_execsql_test 1.5 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 2 {0 1} + 3 0 +} + +#------------------------------------------------------------------------- +# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are +# handled correctly. +# +do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts3 } + +foreach {tn arg} { + 1 {merge=abc} + 2 {merge=%%%} + 3 {merge=,} + 4 {merge=5,} + 5 {merge=6,%} + 6 {merge=6,six} + 7 {merge=6,1} + 8 {merge=6,0} +} { + do_catchsql_test 2.$tn { + INSERT INTO t2(t2) VALUES($arg); + } {1 {SQL logic error or missing database}} +} + +#------------------------------------------------------------------------- +# Test cases 3.* +# +do_test 3.0 { + reset_db + execsql { PRAGMA page_size = 512 } + fts3_build_db_2 30040 +} {} +do_test 3.1 { fts3_integrity_check t2 } {ok} + +do_execsql_test 3.2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6} + 1 {0 1 2 3 4} + 2 {0 1 2 3 4} + 3 {0 1 2 3 4 5 6} +} + +do_execsql_test 3.3 { + INSERT INTO t2(t2) VALUES('merge=1000000,2'); + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level +} { + 0 0 + 2 0 + 3 0 + 4 0 + 6 0 +} + +#------------------------------------------------------------------------- +# Test cases 4.* +# +reset_db +do_execsql_test 4.1 { + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t4 USING fts3; + PRAGMA main.page_size; +} {512} + +do_test 4.2 { + foreach x {a c b d e f g h i j k l m n o p} { + execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" + } + execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} +} {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} + +foreach {tn expect} { + 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" + 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" + 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" + 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" + 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" + 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" + 7 "0 {0 1 2 3 4 5 6 7} 1 0" + 8 "0 {0 1 2 3 4 5 6} 1 0" + 9 "0 {0 1 2 3 4 5} 1 0" +} { + do_execsql_test 4.3.$tn { + INSERT INTO t4(t4) VALUES('merge=1,16'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; + } $expect +} + +do_execsql_test 4.4.1 { + SELECT quote(value) FROM t4_stat WHERE rowid=1 +} {X'0006'} + +do_execsql_test 4.4.2 { + DELETE FROM t4_stat WHERE rowid=1; + INSERT INTO t4(t4) VALUES('merge=1,12'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; +} "0 {0 1 2 3 4 5} 1 0" + + +#------------------------------------------------------------------------- +# Test cases 5.* +# +# Test that if a crisis-merge occurs that disrupts an ongoing incremental +# merge, the next call to "merge=A,B" identifies this and starts a new +# incremental merge. There are two scenarios: +# +# * There are less segments on the input level that the disrupted +# incremental merge operated on, or +# +# * Sufficient segments exist on the input level but the segments +# contain keys smaller than the largest key in the potential output +# segment. +# +do_test 5.1 { + reset_db + fts3_build_db_1 1000 +} {} + +do_execsql_test 5.2 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} + +do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('merge=1,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3} +} + +do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_test 5.5 { + foreach docid [execsql {SELECT docid FROM t1}] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } +} {} + +do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} + +do_execsql_test 5.7 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} + 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 2 {0 1 2 3 4 5 6 7} + X'0104' +} + +do_execsql_test 5.8 { + INSERT INTO t1(t1) VALUES('merge=1,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} + 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 2 {0 1 2 3 4 5 6 7} + 3 {0} + X'0204' +} + +do_test 5.9 { + set L [expr 16*16*8 + 16*4 + 1] + foreach docid [execsql { + SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L + }] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } +} {} + +do_execsql_test 5.10 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 0 1 0 2 0 3 {0 1} + X'0204' +} + +do_execsql_test 5.11 { + INSERT INTO t1(t1) VALUES('merge=10,4'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; +} { + 0 0 1 0 2 0 3 {0 1} + X'0000' +} + +#------------------------------------------------------------------------- +# Test cases 6.* +# +# At one point the following test caused an assert() to fail (because the +# second 'merge=1,2' operation below actually "merges" a single input +# segment, which was unexpected). +# +do_test 6.1 { + reset_db + set a [string repeat a 900] + set b [string repeat b 900] + set c [string repeat c 900] + set d [string repeat d 900] + execsql { + CREATE VIRTUAL TABLE t1 USING fts3; + BEGIN; + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + COMMIT; + } + + execsql { + INSERT INTO t1(t1) VALUES('merge=1,2'); + INSERT INTO t1(t1) VALUES('merge=1,2'); + } +} {} + +#------------------------------------------------------------------------- +# Test cases 7.* +# +# Test that the value returned by sqlite3_total_changes() increases by +# 1 following a no-op "merge=A,B", or by more than 1 if actual work is +# performed. +# +do_test 7.0 { + reset_db + fts3_build_db_1 1000 +} {} + +do_execsql_test 7.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level +} { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} +} +do_test 7.2 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } + expr { ([db total_changes] - $x)>1 } +} {1} +do_test 7.3 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {1} +do_test 7.4 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {0} +do_test 7.5 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } +} {0} + +finish_test From bde1a0b196be3c9f4599c372ffe55bc18cc1e4d9 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Mar 2012 19:44:56 +0000 Subject: [PATCH 37/68] Add SQLITE_DBSTATUS_CACHE_WRITE. Used to query a database connection for the cumulative number of database pages written. FossilOrigin-Name: 3cb6a879f1220db03a66429d63330e27e8ca6e49 --- manifest | 22 +++++++++---------- manifest.uuid | 2 +- src/pager.c | 52 ++++++++++++++++++++++++++++++--------------- src/sqlite.h.in | 14 +++++++++++- src/status.c | 4 +++- src/test_malloc.c | 3 ++- test/dbstatus2.test | 25 +++++++++++++++++++++- 7 files changed, 89 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 3c55a976b3..7fcb982eb0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sfts3\stables\sto\suse\sincremental\smerge\sby\sautomatically\screating\sthe\n%_stat\stable\swhen\sit\sis\sneeded. -D 2012-03-24T17:29:05.827 +C Add\sSQLITE_DBSTATUS_CACHE_WRITE.\sUsed\sto\squery\sa\sdatabase\sconnection\sfor\sthe\scumulative\snumber\sof\sdatabase\spages\swritten. +D 2012-03-24T19:44:56.637 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c 3955b62cdb5bb64559607cb474dd12a6c8e1d4a5 +F src/pager.c 85988507fa20acc60defb834722eddf4633e4aeb F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -182,11 +182,11 @@ F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e -F src/sqlite.h.in 6af2d92925bfed3dfd2c022fef48469762ccd435 +F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a +F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 086dfdd72e5892de223968a258e1ccbd9693e717 F src/test1.c 07f30e8714bab94d8de8a88865d9c59bc512a1a8 @@ -213,7 +213,7 @@ F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee +F src/test_malloc.c 3f5903a1528fd32fe4c472a3bd0259128d8faaef F src/test_multiplex.c 0404a61d7795438be5ee5fd3711eed80724df34d F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e @@ -366,7 +366,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test a18a2ce81add84b17b06559e82ad7bb91bc6ddff F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test dc57b0d9610851c0ff58a8e1b5b191678398b72a +F test/dbstatus2.test b1de8250fde1f3474d6b86f0e89de38d84794f56 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -998,7 +998,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 4220d52cb3426f1680b72d57ecc9f4ade029357d -R d86044022236a45ab8294cd3002625f6 -U drh -Z 5e5822ed626477e734b96980ea604331 +P cc051fc0b2d89603b27b94cf2afdbda417ee9d94 +R 9ca09f80a0e10e381b2361a47e7d01bf +U dan +Z e2a698085c147deaded5cb32de63b51c diff --git a/manifest.uuid b/manifest.uuid index 1019a1397f..2f98caaa3d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc051fc0b2d89603b27b94cf2afdbda417ee9d94 \ No newline at end of file +3cb6a879f1220db03a66429d63330e27e8ca6e49 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 66252aa0f7..2d603122d5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -670,9 +670,9 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int nHit, nMiss; /* Total cache hits and misses */ + int aStat[3]; /* Total cache hits, misses and writes */ #ifdef SQLITE_TEST - int nRead, nWrite; /* Database pages read/written */ + int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC @@ -689,6 +689,15 @@ struct Pager { #endif }; +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + /* ** The following global variables hold counters used for ** testing purposes only. These variables do not exist in @@ -2971,6 +2980,7 @@ static int pagerWalFrames( int isCommit /* True if this is a commit */ ){ int rc; /* Return code */ + int nList; /* Number of pages in pList */ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ #endif @@ -2984,6 +2994,7 @@ static int pagerWalFrames( } #endif + assert( pList->pDirty==0 || isCommit ); if( isCommit ){ /* If a WAL transaction is being committed, there is no point in writing ** any pages with page numbers greater than nTruncate into the WAL file. @@ -2991,11 +3002,18 @@ static int pagerWalFrames( ** list here. */ PgHdr *p; PgHdr **ppNext = &pList; + nList = 0; for(p=pList; (*ppNext = p); p=p->pDirty){ - if( p->pgno<=nTruncate ) ppNext = &p->pDirty; + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } } assert( pList ); + }else{ + nList = 1; } + pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); rc = sqlite3WalFrames(pPager->pWal, @@ -4063,6 +4081,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + pPager->aStat[PAGER_STAT_WRITE]++; /* Update any backup objects copying the contents of this pager. */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); @@ -4071,7 +4090,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } @@ -5029,7 +5047,7 @@ int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - pPager->nHit++; + pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ @@ -5071,7 +5089,7 @@ int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - pPager->nMiss++; + pPager->aStat[PAGER_STAT_MISS]++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -5656,6 +5674,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ pPager->changeCountDone = 1; @@ -6099,11 +6118,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->nWrite; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; return a; } #endif @@ -6116,20 +6135,19 @@ int *sqlite3PagerStats(Pager *pPager){ ** returning. */ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ - int *piStat; assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE ); - if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ - piStat = &pPager->nHit; - }else{ - piStat = &pPager->nMiss; - } - *pnVal += *piStat; + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; if( reset ){ - *piStat = 0; + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 15a1b603a3..2f9c196860 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6001,6 +6001,17 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. ** +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined). ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6012,7 +6023,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 #define SQLITE_DBSTATUS_CACHE_HIT 7 #define SQLITE_DBSTATUS_CACHE_MISS 8 -#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_MAX 9 /* Largest defined DBSTATUS */ /* diff --git a/src/status.c b/src/status.c index b23238bb14..04b7656afe 100644 --- a/src/status.c +++ b/src/status.c @@ -224,10 +224,12 @@ int sqlite3_db_status( ** to zero. */ case SQLITE_DBSTATUS_CACHE_HIT: - case SQLITE_DBSTATUS_CACHE_MISS: { + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; int nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); for(i=0; inDb; i++){ if( db->aDb[i].pBt ){ diff --git a/src/test_malloc.c b/src/test_malloc.c index 037dfd140a..09b8f738e8 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1323,7 +1323,8 @@ static int test_db_status( { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE }, { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL }, { "CACHE_HIT", SQLITE_DBSTATUS_CACHE_HIT }, - { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS } + { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS }, + { "CACHE_WRITE", SQLITE_DBSTATUS_CACHE_WRITE } }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 4dfa9b8ed2..b2ec156655 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# Tests for the sqlite3_stmt_status() function +# Tests for the sqlite3_db_status() function # set testdir [file dirname $argv0] @@ -33,6 +33,10 @@ proc db_hit_miss {db {reset 0}} { list $nHit $nMiss } +proc db_write {db {reset 0}} { + sqlite3_db_status $db CACHE_WRITE $reset +} + do_test 1.1 { db close sqlite3 db test.db @@ -72,5 +76,24 @@ do_test 1.7 { do_test 1.8 { sqlite3_db_status db CACHE_HIT 0 } {0 2 0} do_test 1.9 { sqlite3_db_status db CACHE_MISS 0 } {0 1 0} +do_test 2.1 { db_write db } {0 0 0} +do_test 2.2 { + execsql { INSERT INTO t1 VALUES(4, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.3 { db_write db 1 } {0 4 0} +do_test 2.4 { db_write db 0 } {0 0 0} +do_test 2.5 { db_write db 1 } {0 0 0} + +do_test 2.6 { + execsql { PRAGMA journal_mode = WAL } + db_write db 1 +} {0 1 0} +do_test 2.7 { + execsql { INSERT INTO t1 VALUES(5, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.8 { db_write db 1 } {0 4 0} +do_test 2.9 { db_write db 0 } {0 0 0} finish_test From 9ad3ee40f29ff62fbbb9bb55fd6956d0931e3f72 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 24 Mar 2012 20:06:14 +0000 Subject: [PATCH 38/68] Add SQLITE_DBSTATUS_CACHE_WRITE. Used to query a database connection for the cumulative number of database pages written. FossilOrigin-Name: 05f98d4eec0f029b76fd471f8d9edf2807de6b55 --- manifest | 20 ++++++++--------- manifest.uuid | 2 +- src/pager.c | 52 ++++++++++++++++++++++++++++++--------------- src/sqlite.h.in | 14 +++++++++++- src/status.c | 4 +++- src/test_malloc.c | 3 ++- test/dbstatus2.test | 25 +++++++++++++++++++++- 7 files changed, 88 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 2a0edd6018..898b762958 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sincremental\sblob\scursor\sis\sinvalidated\s(occurs\swhen\san\sSQL\sstatement\smodifies\sor\sdeletes\sthe\srow\sthe\sblob\scursor\spoints\sto)\srelease\sall\spage\sreferences\sheld\sby\sthe\scursor.\sOtherwise,\sthe\spresence\sof\sthese\sreferences\smay\scause\sother\scode\sin\sbtree.c\sto\sincorrectly\sinfer\sthat\sthe\sdatabase\sis\scorrupt. -D 2012-03-23T14:23:52.727 +C Add\sSQLITE_DBSTATUS_CACHE_WRITE.\sUsed\sto\squery\sa\sdatabase\sconnection\sfor\sthe\scumulative\snumber\sof\sdatabase\spages\swritten. +D 2012-03-24T20:06:14.659 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c 3955b62cdb5bb64559607cb474dd12a6c8e1d4a5 +F src/pager.c 85988507fa20acc60defb834722eddf4633e4aeb F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -182,11 +182,11 @@ F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 F src/shell.c 55e09ef7126768b940427d95dc0a8cb7138e95da -F src/sqlite.h.in 6af2d92925bfed3dfd2c022fef48469762ccd435 +F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a +F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 086dfdd72e5892de223968a258e1ccbd9693e717 F src/test1.c 07f30e8714bab94d8de8a88865d9c59bc512a1a8 @@ -213,7 +213,7 @@ F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee +F src/test_malloc.c 3f5903a1528fd32fe4c472a3bd0259128d8faaef F src/test_multiplex.c 0404a61d7795438be5ee5fd3711eed80724df34d F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e @@ -365,7 +365,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test a18a2ce81add84b17b06559e82ad7bb91bc6ddff F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test dc57b0d9610851c0ff58a8e1b5b191678398b72a +F test/dbstatus2.test b1de8250fde1f3474d6b86f0e89de38d84794f56 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 638b71150281a211f89b4057b0d5d32d3fbcf323 -R bf4ea56b709d8c66bd287b12bb865310 +P 341b703ce16361a64ed8bba64ff46792132c0b56 +R 5fc0d309f5291091fecac64a52bbafac U drh -Z 71976091f9202f9aa459dff852dbcf55 +Z 8ee550c76390e8fc0f8077b27e0ba8b4 diff --git a/manifest.uuid b/manifest.uuid index a8830b75fe..f125a8941b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -341b703ce16361a64ed8bba64ff46792132c0b56 \ No newline at end of file +05f98d4eec0f029b76fd471f8d9edf2807de6b55 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 66252aa0f7..2d603122d5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -670,9 +670,9 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int nHit, nMiss; /* Total cache hits and misses */ + int aStat[3]; /* Total cache hits, misses and writes */ #ifdef SQLITE_TEST - int nRead, nWrite; /* Database pages read/written */ + int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC @@ -689,6 +689,15 @@ struct Pager { #endif }; +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + /* ** The following global variables hold counters used for ** testing purposes only. These variables do not exist in @@ -2971,6 +2980,7 @@ static int pagerWalFrames( int isCommit /* True if this is a commit */ ){ int rc; /* Return code */ + int nList; /* Number of pages in pList */ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ #endif @@ -2984,6 +2994,7 @@ static int pagerWalFrames( } #endif + assert( pList->pDirty==0 || isCommit ); if( isCommit ){ /* If a WAL transaction is being committed, there is no point in writing ** any pages with page numbers greater than nTruncate into the WAL file. @@ -2991,11 +3002,18 @@ static int pagerWalFrames( ** list here. */ PgHdr *p; PgHdr **ppNext = &pList; + nList = 0; for(p=pList; (*ppNext = p); p=p->pDirty){ - if( p->pgno<=nTruncate ) ppNext = &p->pDirty; + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } } assert( pList ); + }else{ + nList = 1; } + pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); rc = sqlite3WalFrames(pPager->pWal, @@ -4063,6 +4081,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + pPager->aStat[PAGER_STAT_WRITE]++; /* Update any backup objects copying the contents of this pager. */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); @@ -4071,7 +4090,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } @@ -5029,7 +5047,7 @@ int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - pPager->nHit++; + pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ @@ -5071,7 +5089,7 @@ int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - pPager->nMiss++; + pPager->aStat[PAGER_STAT_MISS]++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -5656,6 +5674,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ pPager->changeCountDone = 1; @@ -6099,11 +6118,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->nWrite; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; return a; } #endif @@ -6116,20 +6135,19 @@ int *sqlite3PagerStats(Pager *pPager){ ** returning. */ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ - int *piStat; assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE ); - if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ - piStat = &pPager->nHit; - }else{ - piStat = &pPager->nMiss; - } - *pnVal += *piStat; + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; if( reset ){ - *piStat = 0; + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 15a1b603a3..2f9c196860 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6001,6 +6001,17 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. ** +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined). ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6012,7 +6023,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 #define SQLITE_DBSTATUS_CACHE_HIT 7 #define SQLITE_DBSTATUS_CACHE_MISS 8 -#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_MAX 9 /* Largest defined DBSTATUS */ /* diff --git a/src/status.c b/src/status.c index b23238bb14..04b7656afe 100644 --- a/src/status.c +++ b/src/status.c @@ -224,10 +224,12 @@ int sqlite3_db_status( ** to zero. */ case SQLITE_DBSTATUS_CACHE_HIT: - case SQLITE_DBSTATUS_CACHE_MISS: { + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; int nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); for(i=0; inDb; i++){ if( db->aDb[i].pBt ){ diff --git a/src/test_malloc.c b/src/test_malloc.c index 037dfd140a..09b8f738e8 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1323,7 +1323,8 @@ static int test_db_status( { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE }, { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL }, { "CACHE_HIT", SQLITE_DBSTATUS_CACHE_HIT }, - { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS } + { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS }, + { "CACHE_WRITE", SQLITE_DBSTATUS_CACHE_WRITE } }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 4dfa9b8ed2..b2ec156655 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# Tests for the sqlite3_stmt_status() function +# Tests for the sqlite3_db_status() function # set testdir [file dirname $argv0] @@ -33,6 +33,10 @@ proc db_hit_miss {db {reset 0}} { list $nHit $nMiss } +proc db_write {db {reset 0}} { + sqlite3_db_status $db CACHE_WRITE $reset +} + do_test 1.1 { db close sqlite3 db test.db @@ -72,5 +76,24 @@ do_test 1.7 { do_test 1.8 { sqlite3_db_status db CACHE_HIT 0 } {0 2 0} do_test 1.9 { sqlite3_db_status db CACHE_MISS 0 } {0 1 0} +do_test 2.1 { db_write db } {0 0 0} +do_test 2.2 { + execsql { INSERT INTO t1 VALUES(4, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.3 { db_write db 1 } {0 4 0} +do_test 2.4 { db_write db 0 } {0 0 0} +do_test 2.5 { db_write db 1 } {0 0 0} + +do_test 2.6 { + execsql { PRAGMA journal_mode = WAL } + db_write db 1 +} {0 1 0} +do_test 2.7 { + execsql { INSERT INTO t1 VALUES(5, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.8 { db_write db 1 } {0 4 0} +do_test 2.9 { db_write db 0 } {0 0 0} finish_test From fbbcd5deda00e70b2739b3cc86bb0c33bddd314a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 24 Mar 2012 20:09:33 +0000 Subject: [PATCH 39/68] Add SQLITE_DBSTATUS_CACHE_WRITE to the command-line shell. FossilOrigin-Name: 30b8dd326d28c0c08543989e376011ea41773a7e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 898b762958..b928d0ee41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_DBSTATUS_CACHE_WRITE.\sUsed\sto\squery\sa\sdatabase\sconnection\sfor\sthe\scumulative\snumber\sof\sdatabase\spages\swritten. -D 2012-03-24T20:06:14.659 +C Add\sSQLITE_DBSTATUS_CACHE_WRITE\sto\sthe\scommand-line\sshell. +D 2012-03-24T20:09:33.655 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 -F src/shell.c 55e09ef7126768b940427d95dc0a8cb7138e95da +F src/shell.c 3179db5d4ff33d62d59a024dbfd2a116390ef7b0 F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 341b703ce16361a64ed8bba64ff46792132c0b56 -R 5fc0d309f5291091fecac64a52bbafac +P 05f98d4eec0f029b76fd471f8d9edf2807de6b55 +R e738d7f44036c20e9a6d260035957782 U drh -Z 8ee550c76390e8fc0f8077b27e0ba8b4 +Z 629ce9245a7ac409b2481f6f2ccdf959 diff --git a/manifest.uuid b/manifest.uuid index f125a8941b..3b15022e45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05f98d4eec0f029b76fd471f8d9edf2807de6b55 \ No newline at end of file +30b8dd326d28c0c08543989e376011ea41773a7e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 9ea5ced282..52b5c4227b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1077,6 +1077,9 @@ static int display_stats( sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + fprintf(pArg->out, "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; From cd217e739722d8cb09885352faac1c95fbeaa738 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 25 Mar 2012 17:25:38 +0000 Subject: [PATCH 40/68] Increase the version number to 3.7.12 FossilOrigin-Name: d95f9fb713c7ba4e570556d835fbd77e574afdea --- VERSION | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index 584a914c29..f7e5aa84c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.11 +3.7.12 diff --git a/manifest b/manifest index b928d0ee41..87409f9e29 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\sSQLITE_DBSTATUS_CACHE_WRITE\sto\sthe\scommand-line\sshell. -D 2012-03-24T20:09:33.655 +C Increase\sthe\sversion\snumber\sto\s3.7.12 +D 2012-03-25T17:25:38.638 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 7849a871b6cdb20fd51baee6bbe5965a03326be4 F Makefile.vxworks 3b7fe7a0571fdadc61363ebc1b23732d2d6363ca F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION bb4c2a86abe53ea3c1d6ea515b69a426040e2414 +F VERSION f9313d88cb77df8617059a88eb382291321ef6bc F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 05f98d4eec0f029b76fd471f8d9edf2807de6b55 -R e738d7f44036c20e9a6d260035957782 +P 30b8dd326d28c0c08543989e376011ea41773a7e +R 9c26fe2aaab0d138abb3ef6a21a226f0 U drh -Z 629ce9245a7ac409b2481f6f2ccdf959 +Z 54613266ba9e6cb2a27d5ad8392e74a5 diff --git a/manifest.uuid b/manifest.uuid index 3b15022e45..5306c9df0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30b8dd326d28c0c08543989e376011ea41773a7e \ No newline at end of file +d95f9fb713c7ba4e570556d835fbd77e574afdea \ No newline at end of file From cbcd9f5357e4f480104f6dc39471663e3cc392da Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Mar 2012 10:36:55 +0000 Subject: [PATCH 41/68] Add an experimental integrity-check function to FTS. FossilOrigin-Name: 40fc8804743dfb005991e9c5ef7b0ebcb3c2e731 --- ext/fts3/fts3.c | 8 +- ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_write.c | 207 +++++++++++++++++++++++++++++++++++++++++ manifest | 21 +++-- manifest.uuid | 2 +- test/fts4check.test | 157 +++++++++++++++++++++++++++++++ test/fts4merge.test | 5 + test/permutations.test | 1 + 8 files changed, 385 insertions(+), 17 deletions(-) create mode 100644 test/fts4check.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 63661393a5..9b7510eead 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2691,7 +2691,7 @@ static int fts3SegReaderCursor( */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ - int iLangid, + int iLangid, /* Language-id to search */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2709,12 +2709,7 @@ int sqlite3Fts3SegReaderCursor( assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); assert( isPrefix==0 || isScan==0 ); - /* "isScan" is only set to true by the ft4aux module, an ordinary - ** full-text tables. */ - assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3MultiSegReader)); - return fts3SegReaderCursor( p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); @@ -5206,6 +5201,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } + /* ** Return SQLITE_CORRUPT_VTAB. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0ce67a2d10..d0ee847ede 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -134,6 +134,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* ** Macro used to suppress compiler warnings for unused parameters. diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 2d3a91d87e..872b1f0d60 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4706,6 +4706,211 @@ static int fts3DoAutoincrmerge( return rc; } +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static i64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + i64 ret = iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + i64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrnIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken; /* Number of bytes in token */ + int iDum1, iDum2; /* Dummy variables */ + int iPos; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + return rc; +} /* ** Handle a 'special' INSERT of the form: @@ -4726,6 +4931,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoOptimize(p, 0); }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ rc = fts3DoIncrmerge(p, &zVal[6]); }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ diff --git a/manifest b/manifest index 7fcb982eb0..418d5baf3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_DBSTATUS_CACHE_WRITE.\sUsed\sto\squery\sa\sdatabase\sconnection\sfor\sthe\scumulative\snumber\sof\sdatabase\spages\swritten. -D 2012-03-24T19:44:56.637 +C Add\san\sexperimental\sintegrity-check\sfunction\sto\sFTS. +D 2012-03-26T10:36:55.434 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 95409b49801ee7736755d7e307e606571b754a58 +F ext/fts3/fts3.c a36f2add4c795b9e1ca1e1a16bd1e45c697a1f37 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h eb749124db7c94b6f89d793cdd4d993a52c46646 +F ext/fts3/fts3Int.h 133e5c613ac6920be5b914d43acc1478df1332e1 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 6a092ee27198716969bfbaa2194aa67eabeb2ff6 +F ext/fts3/fts3_write.c a95e0f29a438bbba69ef686c75f03fbdf7ac79ac F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -497,9 +497,10 @@ F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 +F test/fts4check.test 72134071f4e9f8bed76af1f2375fd5aff0c5ea48 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 120e0baf17a01f0cb696d6f6f9b6506e1587ef90 +F test/fts4merge.test 16ba38960dc06ffd0c47c5487ec1060b5130661f F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -641,7 +642,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 0ab1e7748de5d29c4c648ba5ce3b983ab80653d1 +F test/permutations.test dbda172249564f43ec556108a704581044c57dbd F test/pragma.test c51c148defe32bf4a419a522f95d26838d5cf677 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -998,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cc051fc0b2d89603b27b94cf2afdbda417ee9d94 -R 9ca09f80a0e10e381b2361a47e7d01bf +P 3cb6a879f1220db03a66429d63330e27e8ca6e49 +R a8a5e3d4a755c2fc6211bbe12afb7dcc U dan -Z e2a698085c147deaded5cb32de63b51c +Z 32d9f92fc095c5c84155b7497aaefe53 diff --git a/manifest.uuid b/manifest.uuid index 2f98caaa3d..afcb8fab6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3cb6a879f1220db03a66429d63330e27e8ca6e49 \ No newline at end of file +40fc8804743dfb005991e9c5ef7b0ebcb3c2e731 \ No newline at end of file diff --git a/test/fts4check.test b/test/fts4check.test new file mode 100644 index 0000000000..77815b2ab7 --- /dev/null +++ b/test/fts4check.test @@ -0,0 +1,157 @@ +# 2012 March 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS 'integrity-check' function, +# used to check if the current FTS index accurately reflects the content +# of the table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4check + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +# Run the integrity-check on FTS table $tbl using database handle $db. If +# the integrity-check passes, return "ok". Otherwise, throw an exception. +# +proc fts_integrity {db tbl} { + $db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')" + return "ok" +} + +#------------------------------------------------------------------------- +# Test cases 1.* +# +# 1.0: Build a reasonably sized FTS table (5000 rows). +# +# 1.1: Run the integrity check code to check it passes. +# +# 1.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# + +# Build an FTS table and check the integrity-check passes. +# +do_test 1.0 { fts3_build_db_1 5000 } {} +do_test 1.1 { fts_integrity db t1 } {ok} + +# Mess around with the underlying tables. Check that this causes the +# integrity-check test to fail. +# +foreach {tn disruption} { + 1 { + INSERT INTO t1_content(docid, c0x, c1y) VALUES(NULL, 'a', 'b'); + } + 2 { + DELETE FROM t1_content WHERE docid = (SELECT max(docid) FROM t1_content); + } + 3 { + DELETE FROM t1_segdir WHERE level=0 AND idx=( + SELECT max(idx) FROM t1_segdir WHERE level=0 + ); + } +} { + do_execsql_test 1.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 1.2.2.$tn { + INSERT INTO t1 (t1) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 1.2.3.$tn "ROLLBACK" +} + +do_test 1.3 { fts_integrity db t1 } {ok} + +#------------------------------------------------------------------------- +# Test cases 2.* +# +# 2.0: Build a reasonably sized FTS table (20000 rows) that includes +# prefix indexes. +# +# 2.1: Run the integrity check code to check it passes. +# +# 2.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# + +do_test 2.0 { fts3_build_db_2 20000 {prefix="3,1"} } {} +do_test 2.1 { fts_integrity db t2 } {ok} +foreach {tn disruption} { + 1 { + INSERT INTO t2_content VALUES(NULL, 'xyz') + } + 3 { + DELETE FROM t2_segdir WHERE level=0 AND idx=( + SELECT max(idx) FROM t2_segdir WHERE level=1024 + ); + } +} { + do_execsql_test 2.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 2.2.2.$tn { + INSERT INTO t2 (t2) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 2.2.3.$tn "ROLLBACK" +} + + +#------------------------------------------------------------------------- +# Test cases 3.* +# +# 3.0: Build a reasonably sized FTS table (5000 rows) that includes +# prefix indexes and uses the languageid= feature. +# +# 3.1: Run the integrity check code to check it passes. +# +# 3.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# +do_test 3.0 { + reset_db + fts3_build_db_1 5000 + execsql { + CREATE VIRTUAL TABLE t3 USING fts4(x, y, prefix="2,3", languageid=langid); + } + foreach docid [execsql {SELECT docid FROM t1 ORDER BY 1 ASC}] { + execsql { + INSERT INTO t3(x, y, langid) + SELECT x, y, (docid%9)*4 FROM t1 WHERE docid=$docid; + } + } +} {} +do_test 3.1 { fts_integrity db t3 } {ok} + +foreach {tn disruption} { + 1 { + INSERT INTO t3_content(c0x, c1y, langid) VALUES(NULL, 'a', 0); + } + 2 { + UPDATE t3_content SET langid=langid+1 WHERE rowid = ( + SELECT max(rowid) FROM t3_content + ) + } +} { + do_execsql_test 3.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 3.2.2.$tn { + INSERT INTO t3 (t3) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 3.2.3.$tn "ROLLBACK" +} + + + +finish_test diff --git a/test/fts4merge.test b/test/fts4merge.test index 422d585cbe..95910ea654 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -23,6 +23,11 @@ ifcapable !fts3 { return } +proc fts3_integrity_check {tbl} { + db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')" + return "ok" +} + #------------------------------------------------------------------------- # Test cases 1.* # diff --git a/test/permutations.test b/test/permutations.test index 6f66e85e94..3165ea3eb9 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -185,6 +185,7 @@ test_suite "fts3" -prefix "" -description { fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test + fts4check.test } From 84bce14f773d5a569efc20036d1eb470b85fd028 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Mar 2012 10:47:03 +0000 Subject: [PATCH 42/68] Add a comment to explain how the FTS integrity-check works. FossilOrigin-Name: 64e8a116f39434a3b7347f01a47f88eef3276742 --- ext/fts3/fts3_write.c | 23 +++++++++++++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 872b1f0d60..b0f38b72e8 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4901,6 +4901,29 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ ** ** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite ** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. */ static int fts3DoIntegrityCheck( Fts3Table *p /* FTS3 table handle */ diff --git a/manifest b/manifest index 418d5baf3b..f0ab85f06c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sintegrity-check\sfunction\sto\sFTS. -D 2012-03-26T10:36:55.434 +C Add\sa\scomment\sto\sexplain\show\sthe\sFTS\sintegrity-check\sworks. +D 2012-03-26T10:47:03.798 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c a95e0f29a438bbba69ef686c75f03fbdf7ac79ac +F ext/fts3/fts3_write.c 554368a941e89bf556d653d72eac4ceb8c5a30ef F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 3cb6a879f1220db03a66429d63330e27e8ca6e49 -R a8a5e3d4a755c2fc6211bbe12afb7dcc +P 40fc8804743dfb005991e9c5ef7b0ebcb3c2e731 +R a29b15091e6e8965fd03216fe1c6e4ff U dan -Z 32d9f92fc095c5c84155b7497aaefe53 +Z 4197780953b43ba749fe7b7fcb72124a diff --git a/manifest.uuid b/manifest.uuid index afcb8fab6b..427ed8ce6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40fc8804743dfb005991e9c5ef7b0ebcb3c2e731 \ No newline at end of file +64e8a116f39434a3b7347f01a47f88eef3276742 \ No newline at end of file From 6f4df8ac831ff3726219e237303e80f146f5b22e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Mar 2012 10:57:31 +0000 Subject: [PATCH 43/68] Modify the FTS integrity-check so that the checksums do not depend on the results of signed integer overflow, which is undefined in C. FossilOrigin-Name: f907fc3fb387e74bb66babcbf050748cb253a6fa --- ext/fts3/fts3_write.c | 13 +++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index b0f38b72e8..bb5e515da6 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3611,6 +3611,7 @@ static int fts3IncrmergePush( } assert( 0 ); + return 0; } /* @@ -4710,7 +4711,7 @@ static int fts3DoAutoincrmerge( ** Return a 64-bit checksum for the FTS index entry specified by the ** arguments to this function. */ -static i64 fts3ChecksumEntry( +static u64 fts3ChecksumEntry( const char *zTerm, /* Pointer to buffer containing term */ int nTerm, /* Size of zTerm in bytes */ int iLangid, /* Language id for current row */ @@ -4720,7 +4721,7 @@ static i64 fts3ChecksumEntry( int iPos /* Position */ ){ int i; - i64 ret = iDocid; + u64 ret = (u64)iDocid; ret += (ret<<3) + iLangid; ret += (ret<<3) + iIndex; @@ -4740,7 +4741,7 @@ static i64 fts3ChecksumEntry( ** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The ** return value is undefined in this case. */ -static i64 fts3ChecksumIndex( +static u64 fts3ChecksumIndex( Fts3Table *p, /* FTS3 table handle */ int iLangid, /* Language id to return cksum for */ int iIndex, /* Index to cksum (0..p->nIndex-1) */ @@ -4749,7 +4750,7 @@ static i64 fts3ChecksumIndex( Fts3SegFilter filter; Fts3MultiSegReader csr; int rc; - i64 cksum = 0; + u64 cksum = 0; assert( *pRc==SQLITE_OK ); @@ -4815,8 +4816,8 @@ static i64 fts3ChecksumIndex( */ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ - i64 cksum1 = 0; /* Checksum based on FTS index contents */ - i64 cksum2 = 0; /* Checksum based on %_content contents */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ /* This block calculates the checksum according to the FTS index. */ diff --git a/manifest b/manifest index f0ab85f06c..f8e7923050 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sexplain\show\sthe\sFTS\sintegrity-check\sworks. -D 2012-03-26T10:47:03.798 +C Modify\sthe\sFTS\sintegrity-check\sso\sthat\sthe\schecksums\sdo\snot\sdepend\son\sthe\sresults\sof\ssigned\sinteger\soverflow,\swhich\sis\sundefined\sin\sC. +D 2012-03-26T10:57:31.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 554368a941e89bf556d653d72eac4ceb8c5a30ef +F ext/fts3/fts3_write.c 6014014cf0257d314d29d7eb50e0c88d85356d65 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 40fc8804743dfb005991e9c5ef7b0ebcb3c2e731 -R a29b15091e6e8965fd03216fe1c6e4ff +P 64e8a116f39434a3b7347f01a47f88eef3276742 +R b3b4e0fd75a1473fbbf1af679fdf9753 U dan -Z 4197780953b43ba749fe7b7fcb72124a +Z df6c781d7592577597c979769aed212b diff --git a/manifest.uuid b/manifest.uuid index 427ed8ce6b..98012539c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64e8a116f39434a3b7347f01a47f88eef3276742 \ No newline at end of file +f907fc3fb387e74bb66babcbf050748cb253a6fa \ No newline at end of file From cbf945040412c9c14d6891802914a44dbeea1e5f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Mar 2012 14:36:42 +0000 Subject: [PATCH 44/68] Fix FTS3 so that it works even without SQLITE_DEBUG. FossilOrigin-Name: a18c103121529c2e3c6a8ada16a4c40d14080670 --- ext/fts3/fts3.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 9b7510eead..51db12257e 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3160,7 +3160,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(pVtab); assert( p->pSegments==0 ); assert( p->nPendingData==0 ); diff --git a/manifest b/manifest index f8e7923050..4621b80011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sFTS\sintegrity-check\sso\sthat\sthe\schecksums\sdo\snot\sdepend\son\sthe\sresults\sof\ssigned\sinteger\soverflow,\swhich\sis\sundefined\sin\sC. -D 2012-03-26T10:57:31.278 +C Fix\sFTS3\sso\sthat\sit\sworks\seven\swithout\sSQLITE_DEBUG. +D 2012-03-26T14:36:42.908 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c a36f2add4c795b9e1ca1e1a16bd1e45c697a1f37 +F ext/fts3/fts3.c f2dd409e14eb84ff560a8bfe9c8fc7c93a853c2c F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 133e5c613ac6920be5b914d43acc1478df1332e1 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 64e8a116f39434a3b7347f01a47f88eef3276742 -R b3b4e0fd75a1473fbbf1af679fdf9753 -U dan -Z df6c781d7592577597c979769aed212b +P f907fc3fb387e74bb66babcbf050748cb253a6fa +R c007d11e260643139d517e8dc5ea9d00 +U drh +Z d7453c1ea386bbab453c1602af9f7003 diff --git a/manifest.uuid b/manifest.uuid index 98012539c6..87e220b2bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f907fc3fb387e74bb66babcbf050748cb253a6fa \ No newline at end of file +a18c103121529c2e3c6a8ada16a4c40d14080670 \ No newline at end of file From 36f6b891e6fcf6c7900d65372f790389eefbfbc5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Mar 2012 21:57:53 +0000 Subject: [PATCH 45/68] Add the fts3view utility program. FossilOrigin-Name: f936c8ea16d21345fd1622272dc7e9850acb2493 --- ext/fts3/tool/fts3view.c | 562 +++++++++++++++++++++++++++++++++++++++ manifest | 11 +- manifest.uuid | 2 +- 3 files changed, 569 insertions(+), 6 deletions(-) create mode 100644 ext/fts3/tool/fts3view.c diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c new file mode 100644 index 0000000000..fbda5016cc --- /dev/null +++ b/ext/fts3/tool/fts3view.c @@ -0,0 +1,562 @@ +/* +** This program is a debugging and analysis utility that displays +** information about an FTS3 or FTS4 index. +** +** Link this program against the SQLite3 amalgamation with the +** SQLITE_ENABLE_FTS4 compile-time option. Then run it as: +** +** fts3view DATABASE +** +** to get a list of all FTS3/4 tables in DATABASE, or do +** +** fts3view DATABASE TABLE COMMAND .... +** +** to see various aspects of the TABLE table. Type fts3view with no +** arguments for a list of available COMMANDs. +*/ +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** Extra command-line arguments: +*/ +int nExtra; +char **azExtra; + +/* +** Look for a command-line argument. +*/ +const char *findOption(const char *zName, int hasArg, const char *zDefault){ + int i; + const char *zResult = zDefault; + for(i=0; i=2000 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + if( nDoc>=200 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d most common tokens:\n", nTop); + pStmt = prepare(db, + "SELECT term, documents FROM %s" + " WHERE col='*'" + " ORDER BY documents DESC, term" + " LIMIT %d", zAux, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + n = sqlite3_column_int(pStmt, 1); + printf(" %2d. %-30s %9d docs %5.2f%%\n", i, + sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc); + } + sqlite3_finalize(pStmt); + +end_vocab: + runSql(db, "ROLLBACK"); + sqlite3_free(zAux); +} + +/* +** Report on the number and sizes of segments +*/ +static void showSegmentStats(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + int nSeg = 0; + sqlite3_int64 szSeg = 0, mxSeg = 0; + int nIdx = 0; + sqlite3_int64 szIdx = 0, mxIdx = 0; + int nRoot = 0; + sqlite3_int64 szRoot = 0, mxRoot = 0; + sqlite3_int64 mx; + int nLeaf; + int n; + int pgsz; + int mxLevel; + int i; + + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nSeg = sqlite3_column_int(pStmt, 0); + szSeg = sqlite3_column_int64(pStmt, 1); + mxSeg = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block", + zTab, zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nIdx = sqlite3_column_int(pStmt, 0); + szIdx = sqlite3_column_int64(pStmt, 1); + mxIdx = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(root)), max(length(root))" + " FROM '%q_segdir'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nRoot = sqlite3_column_int(pStmt, 0); + szRoot = sqlite3_column_int64(pStmt, 1); + mxRoot = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + + printf("Number of segments....................... %9d\n", nSeg+nRoot); + printf("Number of leaf segments.................. %9d\n", nSeg-nIdx); + printf("Number of index segments................. %9d\n", nIdx); + printf("Number of root segments.................. %9d\n", nRoot); + printf("Total size of all segments............... %9lld\n", szSeg+szRoot); + printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx); + printf("Total size of all index segments......... %9lld\n", szIdx); + printf("Total size of all root segments.......... %9lld\n", szRoot); + if( nSeg>0 ){ + printf("Average size of all segments............. %11.1f\n", + (double)(szSeg+szRoot)/(double)(nSeg+nRoot)); + printf("Average size of leaf segments............ %11.1f\n", + (double)(szSeg-szIdx)/(double)(nSeg-nIdx)); + } + if( nIdx>0 ){ + printf("Average size of index segments........... %11.1f\n", + (double)szIdx/(double)nIdx); + } + if( nRoot>0 ){ + printf("Average size of root segments............ %11.1f\n", + (double)szRoot/(double)nRoot); + } + mx = mxSeg; + if( mx%d", + zTab, zTab, pgsz-45); + n = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + nLeaf = nSeg - nIdx; + printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + + pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); + mxLevel = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + + for(i=0; i<=mxLevel; i++){ + pStmt = prepare(db, + "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d)," + " count(distinct idx)" + " FROM (SELECT length(a.block) AS len, idx" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE (a.blockid BETWEEN b.start_block" + " AND b.leaves_end_block)" + " AND (b.level%%1024)==%d)", + pgsz-45, zTab, zTab, i); + if( sqlite3_step(pStmt)==SQLITE_ROW + && (nLeaf = sqlite3_column_int(pStmt, 0))>0 + ){ + int nIdx = sqlite3_column_int(pStmt, 5); + sqlite3_int64 sz; + printf("For level %d:\n", i); + printf(" Number of indexes...................... %9d\n", nIdx); + printf(" Number of leaf segments................ %9d\n", nLeaf); + if( nIdx>1 ){ + printf(" Average leaf segments per index........ %11.1f\n", + (double)nLeaf/(double)nIdx); + } + printf(" Total size of all leaf segments........ %9lld\n", + (sz = sqlite3_column_int64(pStmt, 1))); + printf(" Average size of leaf segments.......... %11.1f\n", + sqlite3_column_double(pStmt, 2)); + if( nIdx>1 ){ + printf(" Average leaf segment size per index.... %11.1f\n", + (double)sz/(double)nIdx); + } + printf(" Maximum leaf segment size.............. %9lld\n", + sqlite3_column_int64(pStmt, 3)); + n = sqlite3_column_int(pStmt, 4); + printf(" Leaf segments larger than %5d bytes.. %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + } + sqlite3_finalize(pStmt); + } +} + +/* +** Print a single "tree" line of the segdir map output. +*/ +static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){ + printf(" tree %9lld", iLower); + if( iUpper>iLower ){ + printf(" thru %9lld (%lld blocks)", iUpper, iUpper-iLower+1); + } + printf("\n"); +} + +/* +** Show a map of segments derived from the %_segdir table. +*/ +static void showSegdirMap(sqlite3 *db, const char *zTab){ + int mxIndex, iIndex; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pStmt2 = 0; + int prevLevel; + + pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxIndex = sqlite3_column_int(pStmt, 0); + }else{ + mxIndex = 0; + } + sqlite3_finalize(pStmt); + + printf("Number of inverted indices............... %3d\n", mxIndex+1); + pStmt = prepare(db, + "SELECT level, idx, start_block, leaves_end_block, end_block" + " FROM '%q_segdir'" + " WHERE level/1024==?" + " ORDER BY level DESC, idx", + zTab); + pStmt2 = prepare(db, + "SELECT blockid FROM '%q_segments'" + " WHERE blockid BETWEEN ? AND ? ORDER BY blockid", + zTab); + for(iIndex=0; iIndex<=mxIndex; iIndex++){ + if( mxIndex>0 ){ + printf("**************************** Index %d " + "****************************\n", iIndex); + } + sqlite3_bind_int(pStmt, 1, iIndex); + prevLevel = -1; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int iLevel = sqlite3_column_int(pStmt, 0)%1024; + int iIdx = sqlite3_column_int(pStmt, 1); + sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3); + sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4); + if( iLevel!=prevLevel ){ + printf("level %2d idx %2d", iLevel, iIdx); + prevLevel = iLevel; + }else{ + printf(" idx %2d", iIdx); + } + if( iLEnd>iStart ){ + sqlite3_int64 iLower, iPrev, iX; + printf(" leaves %9lld thru %9lld (%lld blocks)\n", + iStart, iLEnd, iLEnd - iStart + 1); + if( iLEnd+1<=iEnd ){ + sqlite3_bind_int64(pStmt2, 1, iLEnd+1); + sqlite3_bind_int64(pStmt2, 2, iEnd); + iLower = -1; + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + iX = sqlite3_column_int64(pStmt2, 0); + if( iLower<0 ){ + iLower = iPrev = iX; + }else if( iX==iPrev+1 ){ + iPrev = iX; + }else{ + printTreeLine(iLower, iPrev); + iLower = iPrev = iX; + } + } + sqlite3_reset(pStmt2); + if( iLower>=0 ) printTreeLine(iLower, iPrev); + } + }else{ + printf(" root only\n"); + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + sqlite3_finalize(pStmt2); +} + + +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s DATABASE\n" + " or: %s DATABASE FTS3TABLE ARGS...\n", argv0, argv0); + fprintf(stderr, + "ARGS:\n" + " schema FTS table schema\n" + " segdir directory of segments\n" + " segment-stats information about segment sizes\n" + " stat content of the %%_stat table\n" + " vocabulary --top N information on the document vocabulary\n" + ); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db; + int rc; + const char *zTab; + const char *zCmd; + if( argc<2 ) usage(argv[0]); + rc = sqlite3_open(argv[1], &db); + if( rc ){ + fprintf(stderr, "Cannot open %s\n", argv[1]); + exit(1); + } + if( argc==2 ){ + sqlite3_stmt *pStmt; + int cnt = 0; + pStmt = prepare(db, "SELECT b.sql" + " FROM sqlite_master a, sqlite_master b" + " WHERE a.name GLOB '*_segdir'" + " AND b.name=substr(a.name,1,length(a.name)-7)" + " ORDER BY 1"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + cnt++; + printf("%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + if( cnt==0 ){ + printf("/* No FTS3/4 tables found in database %s */\n", argv[1]); + } + return 0; + } + if( argc<4 ) usage(argv[0]); + zTab = argv[2]; + zCmd = argv[3]; + nExtra = argc-4; + azExtra = argv+4; + if( strcmp(zCmd,"schema")==0 ){ + showSchema(db, zTab); + }else if( strcmp(zCmd,"segdir")==0 ){ + showSegdirMap(db, zTab); + }else if( strcmp(zCmd,"segment-stats")==0 ){ + showSegmentStats(db, zTab); + }else if( strcmp(zCmd,"stat")==0 ){ + showStat(db, zTab); + }else if( strcmp(zCmd,"vocabulary")==0 ){ + showVocabulary(db, zTab); + }else{ + usage(argv[0]); + } + return 0; +} diff --git a/manifest b/manifest index 4621b80011..42ffd4a36c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sFTS3\sso\sthat\sit\sworks\seven\swithout\sSQLITE_DEBUG. -D 2012-03-26T14:36:42.908 +C Add\sthe\sfts3view\sutility\sprogram. +D 2012-03-26T21:57:53.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -81,6 +81,7 @@ F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_write.c 6014014cf0257d314d29d7eb50e0c88d85356d65 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 +F ext/fts3/tool/fts3view.c 005efba99f4de1ab104456f7652f955603a81d7f F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -999,7 +1000,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f907fc3fb387e74bb66babcbf050748cb253a6fa -R c007d11e260643139d517e8dc5ea9d00 +P a18c103121529c2e3c6a8ada16a4c40d14080670 +R ad0002e6aa190f63417466651f9099fc U drh -Z d7453c1ea386bbab453c1602af9f7003 +Z 38d7fb610b5e9fceb6186edb4298f4d3 diff --git a/manifest.uuid b/manifest.uuid index 87e220b2bf..2fd4d0cb57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a18c103121529c2e3c6a8ada16a4c40d14080670 \ No newline at end of file +f936c8ea16d21345fd1622272dc7e9850acb2493 \ No newline at end of file From 280d5c33cdeae6e060cb5db18e527ec413fab873 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Mar 2012 00:34:04 +0000 Subject: [PATCH 46/68] Enhance fts3view to show decodes of segments and doclists. FossilOrigin-Name: 6d09de231b68dd9520d99c65d133f26e90eb784f --- ext/fts3/tool/fts3view.c | 260 +++++++++++++++++++++++++++++++++++++-- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 256 insertions(+), 18 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index fbda5016cc..1ecb581171 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "sqlite3.h" /* @@ -436,7 +437,7 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ printf("Number of inverted indices............... %3d\n", mxIndex+1); pStmt = prepare(db, - "SELECT level, idx, start_block, leaves_end_block, end_block" + "SELECT level, idx, start_block, leaves_end_block, end_block, rowid" " FROM '%q_segdir'" " WHERE level/1024==?" " ORDER BY level DESC, idx", @@ -464,10 +465,9 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ }else{ printf(" idx %2d", iIdx); } + printf(" root r%lld\n", sqlite3_column_int64(pStmt, 5)); if( iLEnd>iStart ){ sqlite3_int64 iLower, iPrev, iX; - printf(" leaves %9lld thru %9lld (%lld blocks)\n", - iStart, iLEnd, iLEnd - iStart + 1); if( iLEnd+1<=iEnd ){ sqlite3_bind_int64(pStmt2, 1, iLEnd+1); sqlite3_bind_int64(pStmt2, 2, iEnd); @@ -486,8 +486,8 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ sqlite3_reset(pStmt2); if( iLower>=0 ) printTreeLine(iLower, iPrev); } - }else{ - printf(" root only\n"); + printf(" leaves %9lld thru %9lld (%lld blocks)\n", + iStart, iLEnd, iLEnd - iStart + 1); } } sqlite3_reset(pStmt); @@ -496,17 +496,249 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ sqlite3_finalize(pStmt2); } +/* +** Decode a single segment block and display the results on stdout. +*/ +static void decodeSegment( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iChild; + sqlite3_int64 iPrefix; + sqlite3_int64 nTerm; + sqlite3_int64 n; + sqlite3_int64 iDocsz; + int iHeight; + int i = 0; + int cnt = 0; + char zTerm[1000]; + + i += getVarint(aData, &n); + iHeight = (int)n; + printf("height: %d\n", iHeight); + if( iHeight>0 ){ + i += getVarint(aData+i, &iChild); + printf("left-child: %lld\n", iChild); + } + while( i0 ){ + i += getVarint(aData+i, &iPrefix); + }else{ + iPrefix = 0; + } + i += getVarint(aData+i, &nTerm); + if( iPrefix+nTerm+1 >= sizeof(zTerm) ){ + fprintf(stderr, "term to long\n"); + exit(1); + } + memcpy(zTerm+iPrefix, aData+i, nTerm); + zTerm[iPrefix+nTerm] = 0; + i += nTerm; + if( iHeight==0 ){ + i += getVarint(aData+i, &iDocsz); + printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + i += iDocsz; + }else{ + printf("term: %-25s child %lld\n", zTerm, ++iChild); + } + } +} + + +/* +** Print a a blob as hex and ascii. +*/ +static void printBlob( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + int i, j; + const char *zOfstFmt; + const int perLine = 16; + + if( (nData&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( (nData&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( (nData&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( (nData&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + for(i=0; inData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; jnData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } +} + +/* +** Convert text to a 64-bit integer +*/ +static sqlite3_int64 atoi64(const char *z){ + sqlite3_int64 v = 0; + while( z[0]>='0' && z[0]<='9' ){ + v = v*10 + z[0] - '0'; + z++; + } + return v; +} + +/* +** Return a prepared statement which, when stepped, will return in its +** first column the blob associated with segment zId. If zId begins with +** 'r' then it is a rowid of a %_segdir entry. Otherwise it is a +** %_segment entry. +*/ +static sqlite3_stmt *prepareToGetSegment( + sqlite3 *db, /* The database */ + const char *zTab, /* The FTS3/4 table name */ + const char *zId /* ID of the segment to open */ +){ + sqlite3_stmt *pStmt; + if( zId[0]=='r' ){ + pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld", + zTab, atoi64(zId+1)); + }else{ + pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld", + zTab, atoi64(zId)); + } + return pStmt; +} + +/* +** Print the content of a segment or of the root of a segdir. The segment +** or root is identified by azExtra[0]. If the first character of azExtra[0] +** is 'r' then the remainder is the integer rowid of the %_segdir entry. +** If the first character of azExtra[0] is not 'r' then, then all of +** azExtra[0] is an integer which is the block number. +** +** If the --raw option is present in azExtra, then a hex dump is provided. +** Otherwise a decoding is shown. +*/ +static void showSegment(sqlite3 *db, const char *zTab){ + const unsigned char *aData; + int nData; + sqlite3_stmt *pStmt; + + pStmt = prepareToGetSegment(db, zTab, azExtra[0]); + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + sqlite3_finalize(pStmt); + return; + } + nData = sqlite3_column_bytes(pStmt, 0); + aData = sqlite3_column_blob(pStmt, 0); + printf("Segment %s of size %d bytes:\n", azExtra[0], nData); + if( findOption("raw", 0, 0)!=0 ){ + printBlob(aData, nData); + }else{ + decodeSegment(aData, nData); + } + sqlite3_finalize(pStmt); +} + +/* +** Decode a single doclist and display the results on stdout. +*/ +static void decodeDoclist( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iPrevDocid = 0; + sqlite3_int64 iDocid; + sqlite3_int64 iPos; + sqlite3_int64 iPrevPos = 0; + sqlite3_int64 iCol; + int i = 0; + + while( i Date: Tue, 27 Mar 2012 00:38:33 +0000 Subject: [PATCH 47/68] Minor correct errors in the file format description for FTS3/4 contained in the fts3.c header comment. FossilOrigin-Name: fb8aacdd8fbdc946cb271cc589f76b806387937d --- ext/fts3/fts3.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 51db12257e..626a63d971 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -70,7 +70,7 @@ ** A doclist is stored like this: ** ** array { -** varint docid; +** varint docid; (delta from previous doclist) ** array { (position list for column 0) ** varint position; (2 more than the delta from previous position) ** } @@ -101,8 +101,8 @@ ** at D signals the start of a new column; the 1 at E indicates that the ** new column is column number 1. There are two positions at 12 and 45 ** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the next docid. It has one position 72 (72-2) and then -** terminates with the 0 at K. +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. ** ** A "position-list" is the list of positions for multiple columns for ** a single docid. A "column-list" is the set of positions for a single diff --git a/manifest b/manifest index 21d296b527..80bdaf0fc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sfts3view\sto\sshow\sdecodes\sof\ssegments\sand\sdoclists. -D 2012-03-27T00:34:04.336 +C Minor\scorrect\serrors\sin\sthe\sfile\sformat\sdescription\sfor\sFTS3/4\scontained\sin\nthe\sfts3.c\sheader\scomment. +D 2012-03-27T00:38:33.806 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f2dd409e14eb84ff560a8bfe9c8fc7c93a853c2c +F ext/fts3/fts3.c 5e2d74de50f5b591703923b765e74832621b7c3a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 133e5c613ac6920be5b914d43acc1478df1332e1 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e @@ -1000,7 +1000,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f936c8ea16d21345fd1622272dc7e9850acb2493 -R a7bfe1f32748865cb6cf2cbcda903177 +P 6d09de231b68dd9520d99c65d133f26e90eb784f +R 52fac8e81373757809153ddbb5601b3a U drh -Z fcd448737f1da3e5558671112d888fce +Z 7768dc9c0ca55bddb36c9095acc2e9ee diff --git a/manifest.uuid b/manifest.uuid index 3348c6a1f5..461c127272 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d09de231b68dd9520d99c65d133f26e90eb784f \ No newline at end of file +fb8aacdd8fbdc946cb271cc589f76b806387937d \ No newline at end of file From 5da0aa1603f5003ca7a02ad93009b0ac3b6ab794 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Mar 2012 11:48:02 +0000 Subject: [PATCH 48/68] Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway. FossilOrigin-Name: 7ed9d2f24a650b424b97dfc19b8042c4cf09c82c --- ext/fts3/fts3_write.c | 209 +++++++++++++++++++++++++++--------------- manifest | 19 ++-- manifest.uuid | 2 +- test/fts4merge.test | 40 ++++---- 4 files changed, 166 insertions(+), 104 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index bb5e515da6..640c2078cc 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -328,7 +328,7 @@ static int fts3SqlStmt( ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" - " ORDER BY (level %% 1024) DESC LIMIT 1", + " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See @@ -4448,24 +4448,14 @@ static int fts3IncrmergeChomp( /* ** Store an incr-merge hint in the database. */ -static int fts3IncrmergeHintStore( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to read input data from */ - int nMerge /* Number of segments to merge */ -){ - char aBlob[FTS3_VARINT_MAX * 2]; - int nBlob = 0; - int rc; +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pReplace = 0; - - assert( p->bHasStat ); - nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel); - nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge); + int rc; /* Return code */ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); - sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } @@ -4474,42 +4464,87 @@ static int fts3IncrmergeHintStore( } /* -** Load an incr-merge hint from the database. +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. ** -** The incr-merge hint, if one exists, is stored in the rowid==1 row of -** the %_stat table. +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. */ -static int fts3IncrmergeHintLoad( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 *piAbsLevel, /* Absolute level to read input data from */ - int *pnMerge /* Number of segments to merge */ -){ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pSelect = 0; int rc; - *pnMerge = 0; - *piAbsLevel = 0; - + pHint->n = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); if( rc==SQLITE_OK ){ + int rc2; sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); if( SQLITE_ROW==sqlite3_step(pSelect) ){ const char *aHint = sqlite3_column_blob(pSelect, 0); int nHint = sqlite3_column_bytes(pSelect, 0); if( aHint ){ - int i; - char aBlob[FTS3_VARINT_MAX * 2]; - memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint)); - i = sqlite3Fts3GetVarint(aBlob, piAbsLevel); - sqlite3Fts3GetVarint32(&aBlob[i], pnMerge); + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } } } - rc = sqlite3_reset(pSelect); + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } return rc; } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** @@ -4522,53 +4557,70 @@ static int fts3IncrmergeHintLoad( int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ - int bUseHint = 1; /* True if hint has not yet been attempted */ - sqlite3_int64 iHintAbsLevel = 0;/* Hint level */ - int nHintSeg = 0; /* Hint number of segments */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ int nSeg = 0; /* Number of input segments */ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ - assert( nMin>=2 ); - - rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg); - if( nHintSeg==0 ) bUseHint = 0; + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ - Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ - Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ - IncrmergeWriter *pWriter; /* Writer object */ - const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ - if( bUseHint ){ - iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; - }else{ - sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ - - /* Determine which level to merge segments from. Any level, from any - ** prefix or language index may be selected. Stack variable iAbsLevel - ** is set to the absolute level number of the level to merge from. */ - rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); - if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ - /* There are no levels with nMin or more segments. Or an error has - ** occurred. Either way, exit early. */ - rc = sqlite3_reset(pFindLevel); - iAbsLevel = 0; - nSeg = 0; - break; - } + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = nMin; - sqlite3_reset(pFindLevel); + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } } - /* Allocate space for the cursor, filter and writer objects */ - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); - if( !pWriter ) return SQLITE_NOMEM; - memset(pWriter, 0, nAlloc); - pFilter = (Fts3SegFilter *)&pWriter[1]; - pCsr = (Fts3MultiSegReader *)&pFilter[1]; + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; /* Open a cursor to iterate through the contents of the oldest nSeg ** indexes of absolute level iAbsLevel. If this cursor is opened using @@ -4576,10 +4628,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; - rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); - if( pCsr->nSegment==nSeg && SQLITE_OK==rc - && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ @@ -4606,6 +4661,10 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK ){ nRem -= (1 + pWriter->nWork); rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } } } @@ -4613,15 +4672,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } sqlite3Fts3SegReaderFinish(pCsr); - sqlite3_free(pWriter); - bUseHint = 0; } /* Write the hint values into the %_stat table for the next incr-merger */ - if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){ - rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg); + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); } + sqlite3_free(pWriter); + sqlite3_free(hint.a); return rc; } diff --git a/manifest b/manifest index 80bdaf0fc4..533d49a69a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scorrect\serrors\sin\sthe\sfile\sformat\sdescription\sfor\sFTS3/4\scontained\sin\nthe\sfts3.c\sheader\scomment. -D 2012-03-27T00:38:33.806 +C Allow\smultiple\sincremental\smerges\sto\sproceed\sconcurrently.\sThis\sis\srequired\sto\sprevent\sa\slarge\scrisis-merge\sfrom\soccuring\swhile\san\seven\slarger\sincremental-merge\sis\sunderway. +D 2012-03-27T11:48:02.399 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 6014014cf0257d314d29d7eb50e0c88d85356d65 +F ext/fts3/fts3_write.c a9990753ba132cd79b666c61ec58ae15eb032387 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 71d6149a268e8375cd9e5c8224868fa7460614c3 @@ -501,7 +501,7 @@ F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4check.test 72134071f4e9f8bed76af1f2375fd5aff0c5ea48 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 16ba38960dc06ffd0c47c5487ec1060b5130661f +F test/fts4merge.test c7ee5fbb3b316d1496f1d2fa861c53840b84cef9 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -1000,7 +1000,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 6d09de231b68dd9520d99c65d133f26e90eb784f -R 52fac8e81373757809153ddbb5601b3a -U drh -Z 7768dc9c0ca55bddb36c9095acc2e9ee +P fb8aacdd8fbdc946cb271cc589f76b806387937d +R a481c2564099a87723843f2e40a259a7 +T *branch * fts4-incr-merge-exp +T *sym-fts4-incr-merge-exp * +T -sym-fts4-incr-merge * +U dan +Z 42d22ddb577089e47eaff1e8771c43d5 diff --git a/manifest.uuid b/manifest.uuid index 461c127272..334f6758db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb8aacdd8fbdc946cb271cc589f76b806387937d \ No newline at end of file +7ed9d2f24a650b424b97dfc19b8042c4cf09c82c \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 95910ea654..0a4f6b6184 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -196,47 +196,49 @@ do_execsql_test 5.2 { } do_execsql_test 5.3 { - INSERT INTO t1(t1) VALUES('merge=1,4'); + INSERT INTO t1(t1) VALUES('merge=1,5'); + INSERT INTO t1(t1) VALUES('merge=1,5'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; } { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 0 {0 1 2} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 2 {0 1 2 3} } -do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_test 5.5 { foreach docid [execsql {SELECT docid FROM t1}] { execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} } } {} -do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_execsql_test 5.7 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 0 {0 1 2 3 4 5 6 7 8 9 10} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 2 {0 1 2 3 4 5 6 7} - X'0104' + X'0105' } do_execsql_test 5.8 { - INSERT INTO t1(t1) VALUES('merge=1,4'); + INSERT INTO t1(t1) VALUES('merge=1,6'); + INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} - 2 {0 1 2 3 4 5 6 7} - 3 {0} - X'0204' + 0 {0 1 2 3 4} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3 4 5 6 7 8} X'0106' } +do_test 5.8.1 { fts3_integrity_check t1 } ok + do_test 5.9 { - set L [expr 16*16*8 + 16*4 + 1] + set L [expr 16*16*7 + 16*3 + 12] foreach docid [execsql { SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L }] { @@ -248,17 +250,15 @@ do_execsql_test 5.10 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 0 1 0 2 0 3 {0 1} - X'0204' + 0 0 1 {0 1} 2 0 3 0 X'0106' } do_execsql_test 5.11 { - INSERT INTO t1(t1) VALUES('merge=10,4'); + INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 0 1 0 2 0 3 {0 1} - X'0000' + 0 0 1 {0 1} 2 0 3 0 X'' } #------------------------------------------------------------------------- From 1ecb59a1e5d110b265ec91c4dea92c874e1f6dbe Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Mar 2012 13:51:31 +0000 Subject: [PATCH 49/68] Enhance the fts3view tool with the big-segment command and fix a bug in the display of doclists. FossilOrigin-Name: e9436d8038e5a0d1ba992a77d1064d4a55595f57 --- ext/fts3/tool/fts3view.c | 34 +++++++++++++++++++++++++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 1ecb581171..9b33e15998 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -682,8 +682,8 @@ static void decodeDoclist( printf("\n"); break; }else{ - printf(" %lld", iPrevPos + iPos - 2); - iPrevPos = iPos - 2; + iPrevPos += iPos - 2; + printf(" %lld", iPrevPos); } } } @@ -725,6 +725,31 @@ static void showDoclist(sqlite3 *db, const char *zTab){ sqlite3_finalize(pStmt); } +/* +** Show the top N largest segments +*/ +static void listBigSegments(sqlite3 *db, const char *zTab){ + int nTop, i; + sqlite3_stmt *pStmt; + sqlite3_int64 sz; + sqlite3_int64 id; + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d largest segments:\n", nTop); + pStmt = prepare(db, + "SELECT blockid, length(block) AS len FROM '%q_segments'" + " ORDER BY 2 DESC, 1" + " LIMIT %d", zTab, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + id = sqlite3_column_int64(pStmt, 0); + sz = sqlite3_column_int64(pStmt, 1); + printf(" %2d. %9lld size %lld\n", i, id, sz); + } + sqlite3_finalize(pStmt); +} + static void usage(const char *argv0){ @@ -732,6 +757,7 @@ static void usage(const char *argv0){ " or: %s DATABASE FTS3TABLE ARGS...\n", argv0, argv0); fprintf(stderr, "ARGS:\n" + " big-segments [--top N] show the largest segments\n" " doclist BLOCKID OFFSET SIZE [--raw] Decode a doclist\n" " schema FTS table schema\n" " segdir directory of segments\n" @@ -777,7 +803,9 @@ int main(int argc, char **argv){ zCmd = argv[3]; nExtra = argc-4; azExtra = argv+4; - if( strcmp(zCmd,"doclist")==0 ){ + if( strcmp(zCmd,"big-segments")==0 ){ + listBigSegments(db, zTab); + }else if( strcmp(zCmd,"doclist")==0 ){ if( argc<7 ) usage(argv[0]); showDoclist(db, zTab); }else if( strcmp(zCmd,"schema")==0 ){ diff --git a/manifest b/manifest index 068fa1017a..7da4e728a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sfts4-incr-merge-exp\sbranch\swith\sfts4-incr-merge. -D 2012-03-27T13:44:28.992 +C Enhance\sthe\sfts3view\stool\swith\sthe\sbig-segment\scommand\sand\sfix\sa\sbug\sin\sthe\ndisplay\sof\sdoclists. +D 2012-03-27T13:51:31.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -81,7 +81,7 @@ F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_write.c a9990753ba132cd79b666c61ec58ae15eb032387 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 71d6149a268e8375cd9e5c8224868fa7460614c3 +F ext/fts3/tool/fts3view.c 153b47ecb91856f8462fa8b8320762ad62ca965a F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -1000,7 +1000,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P fb8aacdd8fbdc946cb271cc589f76b806387937d 7ed9d2f24a650b424b97dfc19b8042c4cf09c82c -R a481c2564099a87723843f2e40a259a7 -U dan -Z cbd151378395e54c0cdc970e0e7f2163 +P eb00b95885023a8592693f0babbb0066207073b0 +R 6a5cd3e2138fd11a98667226e3dd3f73 +U drh +Z d39fd2dac9646b20152fd68106b6fe3c diff --git a/manifest.uuid b/manifest.uuid index b6310eca9a..daf1033c00 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb00b95885023a8592693f0babbb0066207073b0 \ No newline at end of file +e9436d8038e5a0d1ba992a77d1064d4a55595f57 \ No newline at end of file From a1a9f0a0d8e7bb0d00ba030a88fb83a56ba34ba1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Mar 2012 14:54:44 +0000 Subject: [PATCH 50/68] In the fts3view utility, label the blank segments used to mark the end of a segment sequence for a level/idx as "null". Improve the alignment of root segment names. FossilOrigin-Name: 04aea0245e4183fef3664609f5a6353b65d71a85 --- ext/fts3/tool/fts3view.c | 31 +++++++++++++++++++++++++++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 9b33e15998..6fa2e40d20 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -418,6 +418,22 @@ static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){ printf("\n"); } +/* +** Check to see if the block of a %_segments entry is NULL. +*/ +static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){ + sqlite3_stmt *pStmt; + int rc = 1; + + pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'" + " WHERE blockid=%lld", zTab, iBlockId); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + rc = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + return rc; +} + /* ** Show a map of segments derived from the %_segdir table. */ @@ -459,13 +475,16 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2); sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3); sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4); + char rtag[20]; if( iLevel!=prevLevel ){ printf("level %2d idx %2d", iLevel, iIdx); prevLevel = iLevel; }else{ printf(" idx %2d", iIdx); } - printf(" root r%lld\n", sqlite3_column_int64(pStmt, 5)); + sqlite3_snprintf(sizeof(rtag), rtag, "r%lld", + sqlite3_column_int64(pStmt,5)); + printf(" root %9s\n", rtag); if( iLEnd>iStart ){ sqlite3_int64 iLower, iPrev, iX; if( iLEnd+1<=iEnd ){ @@ -484,7 +503,15 @@ static void showSegdirMap(sqlite3 *db, const char *zTab){ } } sqlite3_reset(pStmt2); - if( iLower>=0 ) printTreeLine(iLower, iPrev); + if( iLower>=0 ){ + if( iLower==iPrev && iLower==iEnd + && isNullSegment(db,zTab,iLower) + ){ + printf(" null %9lld\n", iLower); + }else{ + printTreeLine(iLower, iPrev); + } + } } printf(" leaves %9lld thru %9lld (%lld blocks)\n", iStart, iLEnd, iLEnd - iStart + 1); diff --git a/manifest b/manifest index 7da4e728a7..e97049c786 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sfts3view\stool\swith\sthe\sbig-segment\scommand\sand\sfix\sa\sbug\sin\sthe\ndisplay\sof\sdoclists. -D 2012-03-27T13:51:31.240 +C In\sthe\sfts3view\sutility,\slabel\sthe\sblank\ssegments\sused\sto\smark\sthe\send\sof\sa\nsegment\ssequence\sfor\sa\slevel/idx\sas\s"null".\s\sImprove\sthe\salignment\sof\sroot\nsegment\snames. +D 2012-03-27T14:54:44.486 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -81,7 +81,7 @@ F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_write.c a9990753ba132cd79b666c61ec58ae15eb032387 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 153b47ecb91856f8462fa8b8320762ad62ca965a +F ext/fts3/tool/fts3view.c 2c7f1bc1feddca85397be174fb6871007c27898b F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -1000,7 +1000,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P eb00b95885023a8592693f0babbb0066207073b0 -R 6a5cd3e2138fd11a98667226e3dd3f73 +P e9436d8038e5a0d1ba992a77d1064d4a55595f57 +R 539ad1cd9b19a07386b41fe8ff2433cc U drh -Z d39fd2dac9646b20152fd68106b6fe3c +Z cd0ad279479f240b2399ce3360a34f55 diff --git a/manifest.uuid b/manifest.uuid index daf1033c00..99d066996b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9436d8038e5a0d1ba992a77d1064d4a55595f57 \ No newline at end of file +04aea0245e4183fef3664609f5a6353b65d71a85 \ No newline at end of file From 311ec02587f9b4ab6a0786261ea301531a1681f9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Mar 2012 15:00:06 +0000 Subject: [PATCH 51/68] Remove the fts3merge.test script in favour of changing the fts4merge.test script so that it runs tests using both fts4 and fts3. Fix some problems with incr-merge and FTS3 tables. FossilOrigin-Name: 5c447e226afca0d46b9ed994dea26a16a9ae168c --- ext/fts3/fts3.c | 6 +- ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_snippet.c | 4 +- ext/fts3/fts3_write.c | 12 +- manifest | 26 +- manifest.uuid | 2 +- test/fts3_common.tcl | 44 ++- test/fts4check.test | 4 +- test/fts4merge.test | 592 ++++++++++++++++++++-------------------- 9 files changed, 366 insertions(+), 325 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 626a63d971..e18f2b10ec 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -642,6 +642,7 @@ static int fts3CreateTables(Fts3Table *p){ p->zDb, p->zName ); } + assert( p->bHasStat==p->bFts4 ); if( p->bHasStat ){ sqlite3Fts3CreateStatTable(&rc, p); } @@ -1284,6 +1285,7 @@ static int fts3InitVtab( p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bFts4 = isFts4; p->bDescIdx = bDescIdx; p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; @@ -2974,7 +2976,7 @@ static int fts3FilterMethod( if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -4385,7 +4387,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); /* Determine which, if any, tokens in the expression should be deferred. */ - if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index d0ee847ede..7d6c34789f 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -209,6 +209,7 @@ struct Fts3Table { char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index fd5bc9786b..dd3706cc80 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -794,8 +794,8 @@ static int fts3MatchinfoCheck( ){ if( (cArg==FTS3_MATCHINFO_NPHRASE) || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 640c2078cc..c0f25dd431 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1516,7 +1516,7 @@ int sqlite3Fts3MsrOvfl( int rc = SQLITE_OK; int pgsz = p->nPgsz; - assert( p->bHasStat ); + assert( p->bFts4 ); assert( pgsz>0 ); for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ @@ -3302,7 +3302,7 @@ static int fts3DoRebuild(Fts3Table *p){ } } } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); } sqlite3_free(aSz); @@ -4729,7 +4729,10 @@ static int fts3DoIncrmerge( rc = SQLITE_ERROR; }else{ rc = SQLITE_OK; - if( !p->bHasStat ) sqlite3Fts3CreateStatTable(&rc, p); + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); } @@ -4754,6 +4757,7 @@ static int fts3DoAutoincrmerge( sqlite3_stmt *pStmt = 0; p->bAutoincrmerge = fts3Getint(&zParam)!=0; if( !p->bHasStat ){ + assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); if( rc ) return rc; } @@ -5347,7 +5351,7 @@ int sqlite3Fts3UpdateMethod( nChng++; } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } diff --git a/manifest b/manifest index e97049c786..a666fe4095 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sfts3view\sutility,\slabel\sthe\sblank\ssegments\sused\sto\smark\sthe\send\sof\sa\nsegment\ssequence\sfor\sa\slevel/idx\sas\s"null".\s\sImprove\sthe\salignment\sof\sroot\nsegment\snames. -D 2012-03-27T14:54:44.486 +C Remove\sthe\sfts3merge.test\sscript\sin\sfavour\sof\schanging\sthe\sfts4merge.test\sscript\sso\sthat\sit\sruns\stests\susing\sboth\sfts4\sand\sfts3.\sFix\ssome\sproblems\swith\sincr-merge\sand\sFTS3\stables. +D 2012-03-27T15:00:06.725 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,22 +63,22 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5e2d74de50f5b591703923b765e74832621b7c3a +F ext/fts3/fts3.c f41f52a24f1a9d7f94291a0e17027e0c28e4f54b F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 133e5c613ac6920be5b914d43acc1478df1332e1 +F ext/fts3/fts3Int.h 5fd2ec4e47faf17bf4a508d6b8ec5fc0f2c80bff F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 -F ext/fts3/fts3_snippet.c c9e126c20760988aa7c43c6ea1379db34738282e +F ext/fts3/fts3_snippet.c 51a3a34c217e24678a133782c1dfb6f2f70fe559 F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6 F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c a9990753ba132cd79b666c61ec58ae15eb032387 +F ext/fts3/fts3_write.c ceb65d6a85f44c7dd1d96f12d04e20f75884bfe3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 2c7f1bc1feddca85397be174fb6871007c27898b @@ -450,7 +450,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 -F test/fts3_common.tcl 52423d0f15bca1d1d2f2b432f0542c4036d655c7 +F test/fts3_common.tcl 99cf6659b87c0f74f55963c2aea03b3a7d66ceb0 F test/fts3aa.test 909d5f530d30a8e36b9328d67285eae6537c79c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 @@ -498,10 +498,10 @@ F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 -F test/fts4check.test 72134071f4e9f8bed76af1f2375fd5aff0c5ea48 +F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test c7ee5fbb3b316d1496f1d2fa861c53840b84cef9 +F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -1000,7 +1000,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e9436d8038e5a0d1ba992a77d1064d4a55595f57 -R 539ad1cd9b19a07386b41fe8ff2433cc -U drh -Z cd0ad279479f240b2399ce3360a34f55 +P 04aea0245e4183fef3664609f5a6353b65d71a85 +R b8c319973c3298c33d22a9e309ade35c +U dan +Z fbf5edaf3f2b7351822253f4d48708a9 diff --git a/manifest.uuid b/manifest.uuid index 99d066996b..d4a6a25e2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04aea0245e4183fef3664609f5a6353b65d71a85 \ No newline at end of file +5c447e226afca0d46b9ed994dea26a16a9ae168c \ No newline at end of file diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index d36ca10989..2ed1f70bf6 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -45,16 +45,29 @@ # #------------------------------------------------------------------------- -# USAGE: fts3_build_db_1 N +# USAGE: fts3_build_db_1 SWITCHES N # # Build a sample FTS table in the database opened by database connection # [db]. The name of the new table is "t1". # -proc fts3_build_db_1 {n} { +proc fts3_build_db_1 {args} { + + set default(-module) fts4 + + set nArg [llength $args] + if {($nArg%2)==0} { + error "wrong # args: should be \"fts3_build_db_1 ?switches? n\"" + } + + set n [lindex $args [expr $nArg-1]] + array set opts [array get default] + array set opts [lrange $args 0 [expr $nArg-2]] + foreach k [array names opts] { + if {0==[info exists default($k)]} { error "unknown option: $k" } + } if {$n > 10000} {error "n must be <= 10000"} - - db eval { CREATE VIRTUAL TABLE t1 USING fts4(x, y) } + db eval "CREATE VIRTUAL TABLE t1 USING $opts(-module) (x, y)" set xwords [list zero one two three four five six seven eight nine ten] set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] @@ -85,12 +98,29 @@ proc fts3_build_db_1 {n} { # Build a sample FTS table in the database opened by database connection # [db]. The name of the new table is "t2". # -proc fts3_build_db_2 {n args} { +proc fts3_build_db_2 {args} { + + set default(-module) fts4 + set default(-extra) "" + + set nArg [llength $args] + if {($nArg%2)==0} { + error "wrong # args: should be \"fts3_build_db_1 ?switches? n\"" + } + + set n [lindex $args [expr $nArg-1]] + array set opts [array get default] + array set opts [lrange $args 0 [expr $nArg-2]] + foreach k [array names opts] { + if {0==[info exists default($k)]} { error "unknown option: $k" } + } if {$n > 100000} {error "n must be <= 100000"} - set sql "CREATE VIRTUAL TABLE t2 USING fts4(content" - foreach a $args { append sql ", " $a } + set sql "CREATE VIRTUAL TABLE t2 USING $opts(-module) (content" + if {$opts(-extra) != ""} { + append sql ", " $opts(-extra) + } append sql ")" db eval $sql diff --git a/test/fts4check.test b/test/fts4check.test index 77815b2ab7..cc1d018aad 100644 --- a/test/fts4check.test +++ b/test/fts4check.test @@ -88,7 +88,7 @@ do_test 1.3 { fts_integrity db t1 } {ok} # this causes the integrity-check code to fail. # -do_test 2.0 { fts3_build_db_2 20000 {prefix="3,1"} } {} +do_test 2.0 { fts3_build_db_2 -extra {prefix="3,1"} 20000 } {} do_test 2.1 { fts_integrity db t2 } {ok} foreach {tn disruption} { 1 { @@ -152,6 +152,4 @@ foreach {tn disruption} { do_execsql_test 3.2.3.$tn "ROLLBACK" } - - finish_test diff --git a/test/fts4merge.test b/test/fts4merge.test index 0a4f6b6184..fabb651e64 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -15,7 +15,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/fts3_common.tcl -set ::testprefix fts4merge # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { @@ -28,308 +27,315 @@ proc fts3_integrity_check {tbl} { return "ok" } -#------------------------------------------------------------------------- -# Test cases 1.* -# -do_test 1.0 { fts3_build_db_1 1004 } {} -do_test 1.1 { fts3_integrity_check t1 } {ok} -do_execsql_test 1.1 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6 7 8 9 10 11} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} -} - -for {set i 0} {$i<20} {incr i} { - do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } - do_test 1.2.$i.2 { fts3_integrity_check t1 } ok - do_execsql_test 1.2.$i.3 { - SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' - } {123 132 213 231 312 321} -} - -do_execsql_test 1.3 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3} - 1 {0 1 2 3 4 5 6} - 2 {0 1 2 3} -} - -for {set i 0} {$i<100} {incr i} { - do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } - do_test 1.4.$i.2 { fts3_integrity_check t1 } ok - do_execsql_test 1.4.$i.3 { - SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' - } {123 132 213 231 312 321} -} - -do_execsql_test 1.5 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 2 {0 1} - 3 0 -} - -#------------------------------------------------------------------------- -# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are -# handled correctly. -# -do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts4 } - -foreach {tn arg} { - 1 {merge=abc} - 2 {merge=%%%} - 3 {merge=,} - 4 {merge=5,} - 5 {merge=6,%} - 6 {merge=6,six} - 7 {merge=6,1} - 8 {merge=6,0} -} { - do_catchsql_test 2.$tn { - INSERT INTO t2(t2) VALUES($arg); - } {1 {SQL logic error or missing database}} -} - -#------------------------------------------------------------------------- -# Test cases 3.* -# -do_test 3.0 { +foreach mod {fts3 fts4} { + set ::testprefix fts4merge-$mod reset_db - execsql { PRAGMA page_size = 512 } - fts3_build_db_2 30040 -} {} -do_test 3.1 { fts3_integrity_check t2 } {ok} -do_execsql_test 3.2 { - SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6} - 1 {0 1 2 3 4} - 2 {0 1 2 3 4} - 3 {0 1 2 3 4 5 6} -} - -do_execsql_test 3.3 { - INSERT INTO t2(t2) VALUES('merge=1000000,2'); - SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level -} { - 0 0 - 2 0 - 3 0 - 4 0 - 6 0 -} - -#------------------------------------------------------------------------- -# Test cases 4.* -# -reset_db -do_execsql_test 4.1 { - PRAGMA page_size = 512; - CREATE VIRTUAL TABLE t4 USING fts4; - PRAGMA main.page_size; -} {512} - -do_test 4.2 { - foreach x {a c b d e f g h i j k l m n o p} { - execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" + #------------------------------------------------------------------------- + # Test cases 1.* + # + do_test 1.0 { fts3_build_db_1 -module $mod 1004 } {} + do_test 1.1 { fts3_integrity_check t1 } {ok} + do_execsql_test 1.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6 7 8 9 10 11} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} } - execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} -} {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} - -foreach {tn expect} { - 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" - 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" - 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" - 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" - 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" - 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" - 7 "0 {0 1 2 3 4 5 6 7} 1 0" - 8 "0 {0 1 2 3 4 5 6} 1 0" - 9 "0 {0 1 2 3 4 5} 1 0" -} { - do_execsql_test 4.3.$tn { - INSERT INTO t4(t4) VALUES('merge=1,16'); + + for {set i 0} {$i<20} {incr i} { + do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } + do_test 1.2.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.2.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} + } + + do_execsql_test 1.3 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3} + 1 {0 1 2 3 4 5 6} + 2 {0 1 2 3} + } + + for {set i 0} {$i<100} {incr i} { + do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } + do_test 1.4.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.4.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} + } + + do_execsql_test 1.5 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 2 {0 1} + 3 0 + } + + #------------------------------------------------------------------------- + # Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are + # handled correctly. + # + do_execsql_test 2.0 "CREATE VIRTUAL TABLE t2 USING $mod" + + foreach {tn arg} { + 1 {merge=abc} + 2 {merge=%%%} + 3 {merge=,} + 4 {merge=5,} + 5 {merge=6,%} + 6 {merge=6,six} + 7 {merge=6,1} + 8 {merge=6,0} + } { + do_catchsql_test 2.$tn { + INSERT INTO t2(t2) VALUES($arg); + } {1 {SQL logic error or missing database}} + } + + #------------------------------------------------------------------------- + # Test cases 3.* + # + do_test 3.0 { + reset_db + execsql { PRAGMA page_size = 512 } + fts3_build_db_2 -module $mod 30040 + } {} + do_test 3.1 { fts3_integrity_check t2 } {ok} + + do_execsql_test 3.2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6} + 1 {0 1 2 3 4} + 2 {0 1 2 3 4} + 3 {0 1 2 3 4 5 6} + } + + do_execsql_test 3.3 { + INSERT INTO t2(t2) VALUES('merge=1000000,2'); + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level + } { + 0 0 + 2 0 + 3 0 + 4 0 + 6 0 + } + + #------------------------------------------------------------------------- + # Test cases 4.* + # + reset_db + do_execsql_test 4.1 " + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t4 USING $mod; + PRAGMA main.page_size; + " {512} + + do_test 4.2 { + foreach x {a c b d e f g h i j k l m n o p} { + execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" + } + execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} + } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} + + foreach {tn expect} { + 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" + 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" + 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" + 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" + 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" + 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" + 7 "0 {0 1 2 3 4 5 6 7} 1 0" + 8 "0 {0 1 2 3 4 5 6} 1 0" + 9 "0 {0 1 2 3 4 5} 1 0" + } { + do_execsql_test 4.3.$tn { + INSERT INTO t4(t4) VALUES('merge=1,16'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; + } $expect + } + + do_execsql_test 4.4.1 { + SELECT quote(value) FROM t4_stat WHERE rowid=1 + } {X'0006'} + + do_execsql_test 4.4.2 { + DELETE FROM t4_stat WHERE rowid=1; + INSERT INTO t4(t4) VALUES('merge=1,12'); SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; - } $expect -} - -do_execsql_test 4.4.1 { - SELECT quote(value) FROM t4_stat WHERE rowid=1 -} {X'0006'} - -do_execsql_test 4.4.2 { - DELETE FROM t4_stat WHERE rowid=1; - INSERT INTO t4(t4) VALUES('merge=1,12'); - SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; -} "0 {0 1 2 3 4 5} 1 0" - - -#------------------------------------------------------------------------- -# Test cases 5.* -# -# Test that if a crisis-merge occurs that disrupts an ongoing incremental -# merge, the next call to "merge=A,B" identifies this and starts a new -# incremental merge. There are two scenarios: -# -# * There are less segments on the input level that the disrupted -# incremental merge operated on, or -# -# * Sufficient segments exist on the input level but the segments -# contain keys smaller than the largest key in the potential output -# segment. -# -do_test 5.1 { - reset_db - fts3_build_db_1 1000 -} {} - -do_execsql_test 5.2 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; -} { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} -} - -do_execsql_test 5.3 { - INSERT INTO t1(t1) VALUES('merge=1,5'); - INSERT INTO t1(t1) VALUES('merge=1,5'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; -} { - 0 {0 1 2} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} - 2 {0 1 2 3} -} - -do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} -do_test 5.5 { - foreach docid [execsql {SELECT docid FROM t1}] { - execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } "0 {0 1 2 3 4 5} 1 0" + + + #------------------------------------------------------------------------- + # Test cases 5.* + # + # Test that if a crisis-merge occurs that disrupts an ongoing incremental + # merge, the next call to "merge=A,B" identifies this and starts a new + # incremental merge. There are two scenarios: + # + # * There are less segments on the input level that the disrupted + # incremental merge operated on, or + # + # * Sufficient segments exist on the input level but the segments + # contain keys smaller than the largest key in the potential output + # segment. + # + do_test 5.1 { + reset_db + fts3_build_db_1 -module $mod 1000 + } {} + + do_execsql_test 5.2 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + } { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} } -} {} - -do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} - -do_execsql_test 5.7 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 {0 1 2 3 4 5 6 7 8 9 10} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} - 2 {0 1 2 3 4 5 6 7} - X'0105' -} - -do_execsql_test 5.8 { - INSERT INTO t1(t1) VALUES('merge=1,6'); - INSERT INTO t1(t1) VALUES('merge=1,6'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 {0 1 2 3 4} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2 3 4 5 6 7 8} X'0106' -} - -do_test 5.8.1 { fts3_integrity_check t1 } ok - -do_test 5.9 { - set L [expr 16*16*7 + 16*3 + 12] - foreach docid [execsql { - SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L - }] { - execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + + do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('merge=1,5'); + INSERT INTO t1(t1) VALUES('merge=1,5'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + } { + 0 {0 1 2} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} + 2 {0 1 2 3} } -} {} - -do_execsql_test 5.10 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 0 1 {0 1} 2 0 3 0 X'0106' -} - -do_execsql_test 5.11 { - INSERT INTO t1(t1) VALUES('merge=1,6'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 0 1 {0 1} 2 0 3 0 X'' -} - -#------------------------------------------------------------------------- -# Test cases 6.* -# -# At one point the following test caused an assert() to fail (because the -# second 'merge=1,2' operation below actually "merges" a single input -# segment, which was unexpected). -# -do_test 6.1 { - reset_db - set a [string repeat a 900] - set b [string repeat b 900] - set c [string repeat c 900] - set d [string repeat d 900] - execsql { - CREATE VIRTUAL TABLE t1 USING fts4; - BEGIN; - INSERT INTO t1 VALUES($a); - INSERT INTO t1 VALUES($b); - COMMIT; - BEGIN; - INSERT INTO t1 VALUES($c); - INSERT INTO t1 VALUES($d); - COMMIT; + + do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} + do_test 5.5 { + foreach docid [execsql {SELECT docid FROM t1}] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } + } {} + + do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} + + do_execsql_test 5.7 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 {0 1 2 3 4 5 6 7 8 9 10} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} + 2 {0 1 2 3 4 5 6 7} + X'0105' } - - execsql { - INSERT INTO t1(t1) VALUES('merge=1,2'); - INSERT INTO t1(t1) VALUES('merge=1,2'); + + do_execsql_test 5.8 { + INSERT INTO t1(t1) VALUES('merge=1,6'); + INSERT INTO t1(t1) VALUES('merge=1,6'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 {0 1 2 3 4} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3 4 5 6 7 8} X'0106' } -} {} + + do_test 5.8.1 { fts3_integrity_check t1 } ok + + do_test 5.9 { + set L [expr 16*16*7 + 16*3 + 12] + foreach docid [execsql { + SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L + }] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } + } {} + + do_execsql_test 5.10 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 0 1 {0 1} 2 0 3 0 X'0106' + } + + do_execsql_test 5.11 { + INSERT INTO t1(t1) VALUES('merge=1,6'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 0 1 {0 1} 2 0 3 0 X'' + } + + #------------------------------------------------------------------------- + # Test cases 6.* + # + # At one point the following test caused an assert() to fail (because the + # second 'merge=1,2' operation below actually "merges" a single input + # segment, which was unexpected). + # + do_test 6.1 { + reset_db + set a [string repeat a 900] + set b [string repeat b 900] + set c [string repeat c 900] + set d [string repeat d 900] -#------------------------------------------------------------------------- -# Test cases 7.* -# -# Test that the value returned by sqlite3_total_changes() increases by -# 1 following a no-op "merge=A,B", or by more than 1 if actual work is -# performed. -# -do_test 7.0 { - reset_db - fts3_build_db_1 1000 -} {} + execsql "CREATE VIRTUAL TABLE t1 USING $mod" + execsql { + BEGIN; + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + COMMIT; + } + + execsql { + INSERT INTO t1(t1) VALUES('merge=1,2'); + INSERT INTO t1(t1) VALUES('merge=1,2'); + } + } {} + + #------------------------------------------------------------------------- + # Test cases 7.* + # + # Test that the value returned by sqlite3_total_changes() increases by + # 1 following a no-op "merge=A,B", or by more than 1 if actual work is + # performed. + # + do_test 7.0 { + reset_db + fts3_build_db_1 -module $mod 1000 + } {} + + do_execsql_test 7.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} + } + do_test 7.2 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } + expr { ([db total_changes] - $x)>1 } + } {1} + do_test 7.3 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {1} + do_test 7.4 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {0} + do_test 7.5 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {0} -do_execsql_test 7.1 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} } -do_test 7.2 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } - expr { ([db total_changes] - $x)>1 } -} {1} -do_test 7.3 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {1} -do_test 7.4 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {0} -do_test 7.5 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {0} finish_test From 3b16fb0423430afbb4e874ddecb8aa8e4fd9174d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Mar 2012 15:10:50 +0000 Subject: [PATCH 52/68] Really delete the fts3merge.test script (should have been deleted by the previous commit). FossilOrigin-Name: 83838149d9dd7956c5f48f760c2f321180d2db5f --- manifest | 11 +- manifest.uuid | 2 +- test/fts3merge.test | 330 -------------------------------------------- 3 files changed, 6 insertions(+), 337 deletions(-) delete mode 100644 test/fts3merge.test diff --git a/manifest b/manifest index a666fe4095..61f584f692 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sfts3merge.test\sscript\sin\sfavour\sof\schanging\sthe\sfts4merge.test\sscript\sso\sthat\sit\sruns\stests\susing\sboth\sfts4\sand\sfts3.\sFix\ssome\sproblems\swith\sincr-merge\sand\sFTS3\stables. -D 2012-03-27T15:00:06.725 +C Really\sdelete\sthe\sfts3merge.test\sscript\s(should\shave\sbeen\sdeleted\sby\sthe\sprevious\scommit). +D 2012-03-27T15:10:50.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -488,7 +488,6 @@ F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 -F test/fts3merge.test acb0be43658029565e7b448f8968149de80549d7 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5 @@ -1000,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 04aea0245e4183fef3664609f5a6353b65d71a85 -R b8c319973c3298c33d22a9e309ade35c +P 5c447e226afca0d46b9ed994dea26a16a9ae168c +R 5d18089d511e28a909d4de9046e3cbed U dan -Z fbf5edaf3f2b7351822253f4d48708a9 +Z f30e1777f0eaeb639d51fafd86db0092 diff --git a/manifest.uuid b/manifest.uuid index d4a6a25e2f..e67e18a6c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c447e226afca0d46b9ed994dea26a16a9ae168c \ No newline at end of file +83838149d9dd7956c5f48f760c2f321180d2db5f \ No newline at end of file diff --git a/test/fts3merge.test b/test/fts3merge.test deleted file mode 100644 index a0854fa1a4..0000000000 --- a/test/fts3merge.test +++ /dev/null @@ -1,330 +0,0 @@ -# 2012 March 06 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file implements regression tests for SQLite library. The -# focus of this script is testing the incremental merge function. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -source $testdir/fts3_common.tcl -set ::testprefix fts3merge - -# If SQLITE_ENABLE_FTS3 is defined, omit this file. -ifcapable !fts3 { - finish_test - return -} - -#------------------------------------------------------------------------- -# Test cases 1.* -# -do_test 1.0 { fts3_build_db_1 1004 } {} -do_test 1.1 { fts3_integrity_check t1 } {ok} -do_execsql_test 1.1 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6 7 8 9 10 11} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} -} - -for {set i 0} {$i<20} {incr i} { - do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } - do_test 1.2.$i.2 { fts3_integrity_check t1 } ok - do_execsql_test 1.2.$i.3 { - SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' - } {123 132 213 231 312 321} -} - -do_execsql_test 1.3 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3} - 1 {0 1 2 3 4 5 6} - 2 {0 1 2 3} -} - -for {set i 0} {$i<100} {incr i} { - do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } - do_test 1.4.$i.2 { fts3_integrity_check t1 } ok - do_execsql_test 1.4.$i.3 { - SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' - } {123 132 213 231 312 321} -} - -do_execsql_test 1.5 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 2 {0 1} - 3 0 -} - -#------------------------------------------------------------------------- -# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are -# handled correctly. -# -do_execsql_test 2.0 { CREATE VIRTUAL TABLE t2 USING fts3 } - -foreach {tn arg} { - 1 {merge=abc} - 2 {merge=%%%} - 3 {merge=,} - 4 {merge=5,} - 5 {merge=6,%} - 6 {merge=6,six} - 7 {merge=6,1} - 8 {merge=6,0} -} { - do_catchsql_test 2.$tn { - INSERT INTO t2(t2) VALUES($arg); - } {1 {SQL logic error or missing database}} -} - -#------------------------------------------------------------------------- -# Test cases 3.* -# -do_test 3.0 { - reset_db - execsql { PRAGMA page_size = 512 } - fts3_build_db_2 30040 -} {} -do_test 3.1 { fts3_integrity_check t2 } {ok} - -do_execsql_test 3.2 { - SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6} - 1 {0 1 2 3 4} - 2 {0 1 2 3 4} - 3 {0 1 2 3 4 5 6} -} - -do_execsql_test 3.3 { - INSERT INTO t2(t2) VALUES('merge=1000000,2'); - SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level -} { - 0 0 - 2 0 - 3 0 - 4 0 - 6 0 -} - -#------------------------------------------------------------------------- -# Test cases 4.* -# -reset_db -do_execsql_test 4.1 { - PRAGMA page_size = 512; - CREATE VIRTUAL TABLE t4 USING fts3; - PRAGMA main.page_size; -} {512} - -do_test 4.2 { - foreach x {a c b d e f g h i j k l m n o p} { - execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" - } - execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} -} {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} - -foreach {tn expect} { - 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" - 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" - 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" - 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" - 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" - 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" - 7 "0 {0 1 2 3 4 5 6 7} 1 0" - 8 "0 {0 1 2 3 4 5 6} 1 0" - 9 "0 {0 1 2 3 4 5} 1 0" -} { - do_execsql_test 4.3.$tn { - INSERT INTO t4(t4) VALUES('merge=1,16'); - SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; - } $expect -} - -do_execsql_test 4.4.1 { - SELECT quote(value) FROM t4_stat WHERE rowid=1 -} {X'0006'} - -do_execsql_test 4.4.2 { - DELETE FROM t4_stat WHERE rowid=1; - INSERT INTO t4(t4) VALUES('merge=1,12'); - SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; -} "0 {0 1 2 3 4 5} 1 0" - - -#------------------------------------------------------------------------- -# Test cases 5.* -# -# Test that if a crisis-merge occurs that disrupts an ongoing incremental -# merge, the next call to "merge=A,B" identifies this and starts a new -# incremental merge. There are two scenarios: -# -# * There are less segments on the input level that the disrupted -# incremental merge operated on, or -# -# * Sufficient segments exist on the input level but the segments -# contain keys smaller than the largest key in the potential output -# segment. -# -do_test 5.1 { - reset_db - fts3_build_db_1 1000 -} {} - -do_execsql_test 5.2 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; -} { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} -} - -do_execsql_test 5.3 { - INSERT INTO t1(t1) VALUES('merge=1,4'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; -} { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2 3} -} - -do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} -do_test 5.5 { - foreach docid [execsql {SELECT docid FROM t1}] { - execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} - } -} {} - -do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} - -do_execsql_test 5.7 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} - 2 {0 1 2 3 4 5 6 7} - X'0104' -} - -do_execsql_test 5.8 { - INSERT INTO t1(t1) VALUES('merge=1,4'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} - 2 {0 1 2 3 4 5 6 7} - 3 {0} - X'0204' -} - -do_test 5.9 { - set L [expr 16*16*8 + 16*4 + 1] - foreach docid [execsql { - SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L - }] { - execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} - } -} {} - -do_execsql_test 5.10 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 0 1 0 2 0 3 {0 1} - X'0204' -} - -do_execsql_test 5.11 { - INSERT INTO t1(t1) VALUES('merge=10,4'); - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; - SELECT quote(value) from t1_stat WHERE rowid=1; -} { - 0 0 1 0 2 0 3 {0 1} - X'0000' -} - -#------------------------------------------------------------------------- -# Test cases 6.* -# -# At one point the following test caused an assert() to fail (because the -# second 'merge=1,2' operation below actually "merges" a single input -# segment, which was unexpected). -# -do_test 6.1 { - reset_db - set a [string repeat a 900] - set b [string repeat b 900] - set c [string repeat c 900] - set d [string repeat d 900] - execsql { - CREATE VIRTUAL TABLE t1 USING fts3; - BEGIN; - INSERT INTO t1 VALUES($a); - INSERT INTO t1 VALUES($b); - COMMIT; - BEGIN; - INSERT INTO t1 VALUES($c); - INSERT INTO t1 VALUES($d); - COMMIT; - } - - execsql { - INSERT INTO t1(t1) VALUES('merge=1,2'); - INSERT INTO t1(t1) VALUES('merge=1,2'); - } -} {} - -#------------------------------------------------------------------------- -# Test cases 7.* -# -# Test that the value returned by sqlite3_total_changes() increases by -# 1 following a no-op "merge=A,B", or by more than 1 if actual work is -# performed. -# -do_test 7.0 { - reset_db - fts3_build_db_1 1000 -} {} - -do_execsql_test 7.1 { - SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level -} { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} - 2 {0 1 2} -} -do_test 7.2 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } - expr { ([db total_changes] - $x)>1 } -} {1} -do_test 7.3 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {1} -do_test 7.4 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {0} -do_test 7.5 { - set x [db total_changes] - execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } - expr { ([db total_changes] - $x)>1 } -} {0} - -finish_test From cdb86dc4374bd1d0c7791e0a8999b249ee8c6eda Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Mar 2012 18:00:05 +0000 Subject: [PATCH 53/68] Add output of PRAGMAs auto_vacuum and encoding to the "schema" command of the fts3view utility program. FossilOrigin-Name: e31076319363a46905836880765bae3bf204ed19 --- ext/fts3/tool/fts3view.c | 23 +++++++++++++++++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 6fa2e40d20..479ae9868d 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -75,14 +75,16 @@ static sqlite3_stmt *prepare(sqlite3 *db, const char *zFormat, ...){ /* ** Run an SQL statement */ -static void runSql(sqlite3 *db, const char *zFormat, ...){ +static int runSql(sqlite3 *db, const char *zFormat, ...){ va_list ap; char *zSql; + int rc; va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); - sqlite3_exec(db, zSql, 0, 0, 0); + rc = sqlite3_exec(db, zSql, 0, 0, 0); va_end(ap); + return rc; } /* @@ -109,6 +111,22 @@ static void showSchema(sqlite3 *db, const char *zTab){ printf("PRAGMA journal_mode=%s;\n", sqlite3_column_text(pStmt, 0)); } sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA auto_vacuum"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zType = "???"; + switch( sqlite3_column_int(pStmt, 0) ){ + case 0: zType = "OFF"; break; + case 1: zType = "FULL"; break; + case 2: zType = "INCREMENTAL"; break; + } + printf("PRAGMA auto_vacuum=%s;\n", zType); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, "PRAGMA encoding"); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("PRAGMA encoding=%s;\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); } /* @@ -801,6 +819,7 @@ int main(int argc, char **argv){ int rc; const char *zTab; const char *zCmd; + if( argc<2 ) usage(argv[0]); rc = sqlite3_open(argv[1], &db); if( rc ){ diff --git a/manifest b/manifest index 61f584f692..94e76c98c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Really\sdelete\sthe\sfts3merge.test\sscript\s(should\shave\sbeen\sdeleted\sby\sthe\sprevious\scommit). -D 2012-03-27T15:10:50.636 +C Add\soutput\sof\sPRAGMAs\sauto_vacuum\sand\sencoding\sto\sthe\s"schema"\scommand\sof\sthe\nfts3view\sutility\sprogram. +D 2012-03-27T18:00:05.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -81,7 +81,7 @@ F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_write.c ceb65d6a85f44c7dd1d96f12d04e20f75884bfe3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 2c7f1bc1feddca85397be174fb6871007c27898b +F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 5c447e226afca0d46b9ed994dea26a16a9ae168c -R 5d18089d511e28a909d4de9046e3cbed -U dan -Z f30e1777f0eaeb639d51fafd86db0092 +P 83838149d9dd7956c5f48f760c2f321180d2db5f +R 1fdbec6e653dc840496748cca8fc7c51 +U drh +Z bdf85c43146250da50e5176dbc98907c diff --git a/manifest.uuid b/manifest.uuid index e67e18a6c4..0d2aaa0b71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83838149d9dd7956c5f48f760c2f321180d2db5f \ No newline at end of file +e31076319363a46905836880765bae3bf204ed19 \ No newline at end of file From a748fdcc43a2f4a5f59acb6c30742d958b2dcac2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 01:34:47 +0000 Subject: [PATCH 54/68] Evaluate typeof(X) and length(Y) where X is any column and Y is a blob column without actually loading X and Y from disk. FossilOrigin-Name: b899dbeb60752843287e2c6ad3577e1d00f0d587 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/delete.c | 2 +- src/expr.c | 38 ++++++++++++++++++++++++++------------ src/func.c | 10 ++++------ src/select.c | 2 +- src/sqliteInt.h | 24 +++++++++++++++++------- src/vdbe.c | 27 ++++++++++++++++++++++----- src/where.c | 2 +- test/func.test | 33 +++++++++++++++++++++++++++++++++ test/pager1.test | 31 ++++++++++++++++++++++++++++--- 11 files changed, 148 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 87409f9e29..c78d6a8b74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.7.12 -D 2012-03-25T17:25:38.638 +C Evaluate\stypeof(X)\sand\slength(Y)\swhere\sX\sis\sany\scolumn\sand\sY\sis\sa\sblob\scolumn\nwithout\sactually\sloading\sX\sand\sY\sfrom\sdisk. +D 2012-03-28T01:34:47.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,11 +133,11 @@ F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 -F src/delete.c 51d32f0a9c880663e54ce309f52e40c325d5e112 -F src/expr.c 00675123e0beec98f999aa4594d2cbe1fec33c1b +F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e +F src/expr.c 86711d201e6ee0e795571900edf20722d008e3ab F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 -F src/func.c e75f41c421f00762ab9da7dc8fb90af3972cf99f +F src/func.c c6b3c94320253a35bda43fb69cc292618e3285d6 F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -180,11 +180,11 @@ F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 +F src/select.c f6f141cb1ea13f1e6564d3e162700e4937baa2a1 F src/shell.c 3179db5d4ff33d62d59a024dbfd2a116390ef7b0 F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 +F src/sqliteInt.h 3756ece33f1e7f8fe2adf8e523566825c809316e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c 32720e873ed0a23e6ee928b676cd995864b984d6 +F src/vdbe.c b91a9135fa4ecd73805cfda7b051dbcc155a0fb8 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -253,7 +253,7 @@ F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 6baab5dfcf4472552c0346d04f6fd2f4f8539c78 +F src/where.c 44d78f5811594065ebbb5354da7f9e6b8b1306d6 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -497,7 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69 -F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca +F test/func.test 6966ad939b8fccc7d48d18e0c1fc8cd1a9f197e6 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 @@ -626,7 +626,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test cf8f40cf77b5c4f762b1e8492390d61b46a81623 +F test/pager1.test eb6d64d2e148dc4bfc6420605bee3717e2d5f10c F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 30b8dd326d28c0c08543989e376011ea41773a7e -R 9c26fe2aaab0d138abb3ef6a21a226f0 +P d95f9fb713c7ba4e570556d835fbd77e574afdea +R 89de5eb864ab0dd31529150cd7eba40a U drh -Z 54613266ba9e6cb2a27d5ad8392e74a5 +Z 999e726acb9e2120206bfa344ce7af80 diff --git a/manifest.uuid b/manifest.uuid index 5306c9df0b..e1ec8508a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95f9fb713c7ba4e570556d835fbd77e574afdea \ No newline at end of file +b899dbeb60752843287e2c6ad3577e1d00f0d587 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index f666b90c83..eead4856b1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -374,7 +374,7 @@ void sqlite3DeleteFrom( pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK ); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); diff --git a/src/expr.c b/src/expr.c index 79dd8f4961..b76a8afe12 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2032,15 +2032,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -2175,7 +2166,8 @@ int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -2190,7 +2182,11 @@ int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -2318,7 +2314,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; }else{ inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + pExpr->iColumn, pExpr->iTable, target, + pExpr->op2); } break; } @@ -2595,6 +2592,23 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + if( pFarg->a[0].pExpr->op==TK_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->flags==SQLITE_FUNC_LENGTH ); + pFarg->a[0].pExpr->op2 = pDef->flags; + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ diff --git a/src/func.c b/src/func.c index c66ad28aba..6ffc7184b0 100644 --- a/src/func.c +++ b/src/func.c @@ -1542,8 +1542,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), @@ -1555,11 +1555,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/select.c b/src/select.c index 3efe014d8b..c225013672 100644 --- a/src/select.c +++ b/src/select.c @@ -4228,7 +4228,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 15c3f302e2..deb576d2d7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1009,14 +1009,18 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ #define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ #define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_COUNT 0x10 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_LENGTH 0x40 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x80 /* Built-in typeof() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1044,7 +1048,10 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ @@ -1667,6 +1674,7 @@ struct Expr { i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ + /* If TK_COLUMN, the value of p5 for OP_Column */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ #if SQLITE_MAX_EXPR_DEPTH>0 @@ -1689,7 +1697,7 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Hint 0x1000 /* Optimizer hint. Not required for correctness */ +#define EP_Hint 0x1000 /* Not used */ #define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ #define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ @@ -2276,7 +2284,7 @@ struct AuthContext { }; /* -** Bitfield flags for P5 value in OP_Insert and OP_Delete +** Bitfield flags for P5 value in various opcodes. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ @@ -2284,6 +2292,8 @@ struct AuthContext { #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ /* * Each trigger present in the database schema is stored as an instance of @@ -2767,7 +2777,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); diff --git a/src/vdbe.c b/src/vdbe.c index 749830bc0d..93032936ae 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2127,6 +2127,11 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** then the cache of the cursor is reset prior to extracting the column. ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG bit is set on P5 then the result is guaranteed +** to only be used as the argument of a length() or typeof() function and +** so loading of large blobs and strings can be skipped - all that is necessary +** is that the size and type information be set. */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2380,12 +2385,24 @@ case OP_Column: { sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ len = sqlite3VdbeSerialTypeLen(aType[p2]); - sqlite3VdbeMemMove(&sMem, pDest); - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; + if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && (((t = aType[p2])>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(x) function is x is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. */ + zData = t<12 ? (char*)&payloadSize64 : 0; + }else{ + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + zData = sMem.z; } - zData = sMem.z; sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); } pDest->enc = encoding; diff --git a/src/where.c b/src/where.c index e6083f0033..2f3dc77e12 100644 --- a/src/where.c +++ b/src/where.c @@ -4383,7 +4383,7 @@ static Bitmask codeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); + regRowid, 0); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } diff --git a/test/func.test b/test/func.test index eef0543986..37631a419f 100644 --- a/test/func.test +++ b/test/func.test @@ -1247,4 +1247,37 @@ do_test func-28.1 { } } {1 {unknown function: nosuchfunc()}} +# Verify that the length() and typeof() functions do not actually load +# the content of their argument. +# +do_test func-29.1 { + db eval { + CREATE TABLE t29(id INTEGER PRIMARY KEY, x, y); + INSERT INTO t29 VALUES(1, 2, 3), (2, NULL, 4), (3, 4.5, 5); + INSERT INTO t29 VALUES(4, randomblob(1000000), 6); + INSERT INTO t29 VALUES(5, "hello", 7); + } + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(x), length(x), typeof(y) FROM t29 ORDER BY id} +} {integer 1 integer null {} integer real 3 integer blob 1000000 integer text 5 integer} +do_test func-29.2 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} +do_test func-29.3 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(+x) FROM t29 ORDER BY id} +} {integer null real blob text} +do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x +} {many} + + finish_test diff --git a/test/pager1.test b/test/pager1.test index 45253857c3..2ac56dec8c 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -1768,7 +1768,7 @@ do_test pager1-18.2 { catchsql { SELECT count(*) FROM t1 } db2 } {1 {database disk image is malformed}} db2 close -do_test pager1-18.3 { +do_test pager1-18.3.1 { execsql { CREATE TABLE t2(x); INSERT INTO t2 VALUES(a_string(5000)); @@ -1776,13 +1776,38 @@ do_test pager1-18.3 { set pgno [expr ([file size test.db] / 1024)-2] hexio_write test.db [expr ($pgno-1)*1024] 00000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + # even though x is malformed, because typeof() does + # not load the content of x, the error is not noticed. + catchsql { SELECT typeof(x) FROM t2 } db2 +} {0 text} +do_test pager1-18.3.2 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 +} {1 {database disk image is malformed}} +db2 close +do_test pager1-18.3.3 { + execsql { + DELETE FROM t2; + INSERT INTO t2 VALUES(randomblob(5000)); + } + set pgno [expr ([file size test.db] / 1024)-2] + hexio_write test.db [expr ($pgno-1)*1024] 00000000 + sqlite3 db2 test.db + # even though x is malformed, because length() and typeof() do + # not load the content of x, the error is not noticed. + catchsql { SELECT length(x), typeof(x) FROM t2 } db2 +} {0 {5000 blob}} +do_test pager1-18.3.4 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.4 { hexio_write test.db [expr ($pgno-1)*1024] 90000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.5 { From 31ec740a3dfc1190ed972fba8cb99e4de7db0254 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 02:43:20 +0000 Subject: [PATCH 55/68] Fix the typeof() and length() optimization so that it works for aggregates as well as scalar queries. FossilOrigin-Name: bc18215a8a660442db6ddeeda4a88df0acffe0f7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c78d6a8b74..83178b14b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Evaluate\stypeof(X)\sand\slength(Y)\swhere\sX\sis\sany\scolumn\sand\sY\sis\sa\sblob\scolumn\nwithout\sactually\sloading\sX\sand\sY\sfrom\sdisk. -D 2012-03-28T01:34:47.425 +C Fix\sthe\stypeof()\sand\slength()\soptimization\sso\sthat\sit\sworks\sfor\saggregates\nas\swell\sas\sscalar\squeries. +D 2012-03-28T02:43:20.877 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,7 +134,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e -F src/expr.c 86711d201e6ee0e795571900edf20722d008e3ab +F src/expr.c 1c351f385950a40b0283328eb925641a3aa50a84 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 F src/func.c c6b3c94320253a35bda43fb69cc292618e3285d6 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d95f9fb713c7ba4e570556d835fbd77e574afdea -R 89de5eb864ab0dd31529150cd7eba40a +P b899dbeb60752843287e2c6ad3577e1d00f0d587 +R f4c337d261ff97dcd84dc2dd2a1e0095 U drh -Z 999e726acb9e2120206bfa344ce7af80 +Z 615304a5ba3b27e6b38911664d03f093 diff --git a/manifest.uuid b/manifest.uuid index e1ec8508a4..93fa65d24b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b899dbeb60752843287e2c6ad3577e1d00f0d587 \ No newline at end of file +bc18215a8a660442db6ddeeda4a88df0acffe0f7 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index b76a8afe12..1915454742 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2599,9 +2599,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** loading. */ if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 op; assert( nFarg==1 ); assert( pFarg->a[0].pExpr!=0 ); - if( pFarg->a[0].pExpr->op==TK_COLUMN ){ + op = pFarg->a[0].pExpr->op; + if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); testcase( pDef->flags==SQLITE_FUNC_LENGTH ); From 3c888b7d1b39f1e08bc2cf627ec7b4ec3b98b63e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 02:51:51 +0000 Subject: [PATCH 56/68] Test cases for length() of a large blob in an aggregate query. FossilOrigin-Name: d095fa4bfabd765c8e935ed227a334161097dd34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/func.test | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 83178b14b7..398f5477a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stypeof()\sand\slength()\soptimization\sso\sthat\sit\sworks\sfor\saggregates\nas\swell\sas\sscalar\squeries. -D 2012-03-28T02:43:20.877 +C Test\scases\sfor\slength()\sof\sa\slarge\sblob\sin\san\saggregate\squery. +D 2012-03-28T02:51:51.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -497,7 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69 -F test/func.test 6966ad939b8fccc7d48d18e0c1fc8cd1a9f197e6 +F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P b899dbeb60752843287e2c6ad3577e1d00f0d587 -R f4c337d261ff97dcd84dc2dd2a1e0095 +P bc18215a8a660442db6ddeeda4a88df0acffe0f7 +R 3ecaca7bd152423dde5d0467891373c7 U drh -Z 615304a5ba3b27e6b38911664d03f093 +Z c57e0152e24243ecb61f7c7f2f11b01f diff --git a/manifest.uuid b/manifest.uuid index 93fa65d24b..2f703b38c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc18215a8a660442db6ddeeda4a88df0acffe0f7 \ No newline at end of file +d095fa4bfabd765c8e935ed227a334161097dd34 \ No newline at end of file diff --git a/test/func.test b/test/func.test index 37631a419f..ba1ea026d7 100644 --- a/test/func.test +++ b/test/func.test @@ -1278,6 +1278,17 @@ do_test func-29.4 { if {$x>100} {set x many} set x } {many} +do_test func-29.5 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT sum(length(x)) FROM t29} +} {1000009} +do_test func-29.6 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} finish_test From dda5c08bf629e982318eac23dae51827fd39d391 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 13:41:10 +0000 Subject: [PATCH 57/68] Improvements to comments. Minor changes to code in the hot path of OP_Column - with the hope of get a few cycles of performance improvement. FossilOrigin-Name: ca093103437f141caa3eb11539c6eb7b4dd65175 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 25 +++++++++++++------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 398f5477a1..d2e78d5c20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sfor\slength()\sof\sa\slarge\sblob\sin\san\saggregate\squery. -D 2012-03-28T02:51:51.807 +C Improvements\sto\scomments.\s\sMinor\schanges\sto\scode\sin\sthe\shot\spath\sof\nOP_Column\s-\swith\sthe\shope\sof\sget\sa\sfew\scycles\sof\sperformance\simprovement. +D 2012-03-28T13:41:10.884 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c b91a9135fa4ecd73805cfda7b051dbcc155a0fb8 +F src/vdbe.c 807532457744a264d19e929b2075dbec90d2a36d F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bc18215a8a660442db6ddeeda4a88df0acffe0f7 -R 3ecaca7bd152423dde5d0467891373c7 +P d095fa4bfabd765c8e935ed227a334161097dd34 +R 78ef170f6dad0ba0a0f49e941cd533cb U drh -Z c57e0152e24243ecb61f7c7f2f11b01f +Z 6f108b9c2dd42e83ba81e7472cbc6ad3 diff --git a/manifest.uuid b/manifest.uuid index 2f703b38c1..3d77c17c7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d095fa4bfabd765c8e935ed227a334161097dd34 \ No newline at end of file +ca093103437f141caa3eb11539c6eb7b4dd65175 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 93032936ae..fa8dcdf1b7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2128,10 +2128,10 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. ** -** If the OPFLAG_LENGTHARG bit is set on P5 then the result is guaranteed -** to only be used as the argument of a length() or typeof() function and -** so loading of large blobs and strings can be skipped - all that is necessary -** is that the size and type information be set. +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2272,7 +2272,7 @@ case OP_Column: { pC->aRow = 0; } } - /* The following assert is true in all cases accept when + /* The following assert is true in all cases except when ** the database file has been corrupted externally. ** assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */ szHdr = getVarint32((u8*)zData, offset); @@ -2347,11 +2347,11 @@ case OP_Column: { break; } }else{ - /* If i is less that nField, then there are less fields in this + /* If i is less that nField, then there are fewer fields in this ** record than SetNumColumns indicated there are columns in the ** table. Set the offset for any extra columns not present in - ** the record to 0. This tells code below to store a NULL - ** instead of deserializing a value from the record. + ** the record to 0. This tells code below to store the default value + ** for the column instead of deserializing a value from the record. */ aOffset[i] = 0; } @@ -2384,12 +2384,13 @@ case OP_Column: { VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ - len = sqlite3VdbeSerialTypeLen(aType[p2]); + t = aType[p2]; + len = sqlite3VdbeSerialTypeLen(t); if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 - && (((t = aType[p2])>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) ){ /* Content is irrelevant for the typeof() function and for - ** the length(x) function is x is a blob. So we might as well use + ** the length(X) function if X is a blob. So we might as well use ** bogus content rather than reading content from disk. NULL works ** for text and blob and whatever is in the payloadSize64 variable ** will work for everything else. */ @@ -2403,7 +2404,7 @@ case OP_Column: { } zData = sMem.z; } - sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); + sqlite3VdbeSerialGet((u8*)zData, t, pDest); } pDest->enc = encoding; }else{ From eef7445d024d2b2895df2a146a2c1521fbc5b246 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Mar 2012 13:55:29 +0000 Subject: [PATCH 58/68] Fix a problem in fts4merge3.test. FossilOrigin-Name: 64fc8b30f8bc7ddc697b6042040e958787ff3118 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts4merge3.test | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 94e76c98c2..a0020861be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soutput\sof\sPRAGMAs\sauto_vacuum\sand\sencoding\sto\sthe\s"schema"\scommand\sof\sthe\nfts3view\sutility\sprogram. -D 2012-03-27T18:00:05.762 +C Fix\sa\sproblem\sin\sfts4merge3.test. +D 2012-03-28T13:55:29.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -502,7 +502,7 @@ F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 -F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb +F test/fts4merge3.test 7ca2225996f6c4e12efbd4fe34ce8a805adecfb2 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 83838149d9dd7956c5f48f760c2f321180d2db5f -R 1fdbec6e653dc840496748cca8fc7c51 -U drh -Z bdf85c43146250da50e5176dbc98907c +P e31076319363a46905836880765bae3bf204ed19 +R 8261f69797f0415c0a684d0e5b4da417 +U dan +Z 6d2d05b1946beea93ba45e372e1569b3 diff --git a/manifest.uuid b/manifest.uuid index 0d2aaa0b71..714c488ce6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e31076319363a46905836880765bae3bf204ed19 \ No newline at end of file +64fc8b30f8bc7ddc697b6042040e958787ff3118 \ No newline at end of file diff --git a/test/fts4merge3.test b/test/fts4merge3.test index 8cd3cf250c..86b44e5eee 100644 --- a/test/fts4merge3.test +++ b/test/fts4merge3.test @@ -53,7 +53,7 @@ do_all_bc_test { do_test 1.7 { sql2 { SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 - } } [list 0 15 1 1 2 14 3 4 4 1] + } } [list 0 1 2 18 3 5] # Using the old connection, insert many rows. do_test 1.8 { @@ -64,7 +64,7 @@ do_all_bc_test { do_test 1.9 { sql2 { SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 - } } {0 11 1 15 2 3 3 5 4 1} + } } [list 0 13 1 13 2 5 3 6] # Run a big incr-merge operation on the db. do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {} From ac5e749be8997bd2d216eccd675ebe7ddbd2a888 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 16:14:50 +0000 Subject: [PATCH 59/68] Minor changes to the core of OP_Column for performance and to clarify the critical path. FossilOrigin-Name: 868394761e41b7483a5874426ee052dfb3a9e2be --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d2e78d5c20..36e1985ccd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments.\s\sMinor\schanges\sto\scode\sin\sthe\shot\spath\sof\nOP_Column\s-\swith\sthe\shope\sof\sget\sa\sfew\scycles\sof\sperformance\simprovement. -D 2012-03-28T13:41:10.884 +C Minor\schanges\sto\sthe\score\sof\sOP_Column\sfor\sperformance\sand\sto\sclarify\sthe\ncritical\spath. +D 2012-03-28T16:14:50.602 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c 807532457744a264d19e929b2075dbec90d2a36d +F src/vdbe.c 8913926230bfc9d183fcd44e3d023c6d393b6548 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d095fa4bfabd765c8e935ed227a334161097dd34 -R 78ef170f6dad0ba0a0f49e941cd533cb +P ca093103437f141caa3eb11539c6eb7b4dd65175 +R 14c36b139bb9a241f9fc5fc1fa96e28f U drh -Z 6f108b9c2dd42e83ba81e7472cbc6ad3 +Z 95f518d28e036a56a444945b5630fdfa diff --git a/manifest.uuid b/manifest.uuid index 3d77c17c7f..d80571e66d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca093103437f141caa3eb11539c6eb7b4dd65175 \ No newline at end of file +868394761e41b7483a5874426ee052dfb3a9e2be \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index fa8dcdf1b7..9daa9cbff6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2381,11 +2381,12 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ + /* This is the common case where the whole row fits on a single page */ VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ + /* This branch happens only when the row overflows onto multiple pages */ t = aType[p2]; - len = sqlite3VdbeSerialTypeLen(t); if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) ){ @@ -2396,6 +2397,7 @@ case OP_Column: { ** will work for everything else. */ zData = t<12 ? (char*)&payloadSize64 : 0; }else{ + len = sqlite3VdbeSerialTypeLen(t); sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); From 9ace112c84f8fc9e4366edb3facb8b95f27d2628 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Mar 2012 07:51:45 +0000 Subject: [PATCH 60/68] Fix an out of date comment on sqlite3ArrayAllocate(). FossilOrigin-Name: 4afdd5ae53ef0ff7c0fde74eaa04638c923c679b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 22 ++++++++++++---------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index de13e6de0f..508e95c5af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sloading\soverflow\spages\sjust\sto\ssatisfy\stypeof()\sor\slength()\sfunctions\nif\sthe\scorrect\sresult\scan\sbe\scomputed\swithout\sthe\sextra\spage\sfetches. -D 2012-03-28T16:22:03.615 +C Fix\san\sout\sof\sdate\scomment\son\ssqlite3ArrayAllocate(). +D 2012-03-29T07:51:45.964 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -128,7 +128,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 02aeee1f6d425e11f7b9b2d9d461ac501645ed6f F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c -F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 +F src/build.c 139dc386ebdaa78d2198247ebcf003305a5babe3 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d95f9fb713c7ba4e570556d835fbd77e574afdea 868394761e41b7483a5874426ee052dfb3a9e2be -R 14c36b139bb9a241f9fc5fc1fa96e28f -U drh -Z 15a5058e7b3160b8bbe377cfe96ccfbc +P 0733c98c329bc9942460746e9bbaf4b4c94c1520 +R cdd95431de81f865d03de0c1a140cd52 +U dan +Z 5c9d2afc31444fe45f09b85b23502baf diff --git a/manifest.uuid b/manifest.uuid index 60c0fc890f..48ec1465b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0733c98c329bc9942460746e9bbaf4b4c94c1520 \ No newline at end of file +4afdd5ae53ef0ff7c0fde74eaa04638c923c679b \ No newline at end of file diff --git a/src/build.c b/src/build.c index daa5430406..46b280a70c 100644 --- a/src/build.c +++ b/src/build.c @@ -3040,19 +3040,21 @@ exit_drop_index: } /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. ** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). ** -** The index of the new entry is returned in *pIdx. +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. */ void *sqlite3ArrayAllocate( sqlite3 *db, /* Connection to notify of malloc failures */ From c68939ef3e403e8996a11f5805a9bb12415318c8 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Mar 2012 14:29:07 +0000 Subject: [PATCH 61/68] Disable the LIKE optimization if the column on the left-hand-side of the LIKE operator belongs to a virtual table. FossilOrigin-Name: 0bacb879e18026f2a8e22fe3e4bc8d27de5c4416 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/where.c | 5 ++++- test/vtab1.test | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 508e95c5af..3caf718a94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sout\sof\sdate\scomment\son\ssqlite3ArrayAllocate(). -D 2012-03-29T07:51:45.964 +C Disable\sthe\sLIKE\soptimization\sif\sthe\scolumn\son\sthe\sleft-hand-side\sof\sthe\sLIKE\soperator\sbelongs\sto\sa\svirtual\stable. +D 2012-03-29T14:29:07.705 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -253,7 +253,7 @@ F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 44d78f5811594065ebbb5354da7f9e6b8b1306d6 +F src/where.c 2112422a404dcca5d47f6630bdf180bccd36c62b F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -890,7 +890,7 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 7701bb609fe8bf6535514e8b849a309e8f00573b F test/view.test b182a67ec43f490b156b5a710827a341be83dd17 -F test/vtab1.test 17d0db1096b603a357f943a9dcbdc3e30c6f04f2 +F test/vtab1.test e429a6835faa3870016c55d1178dcfead85f936a F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 0733c98c329bc9942460746e9bbaf4b4c94c1520 -R cdd95431de81f865d03de0c1a140cd52 +P 4afdd5ae53ef0ff7c0fde74eaa04638c923c679b +R def111c8a7bf40297c8914cb6a05a57c U dan -Z 5c9d2afc31444fe45f09b85b23502baf +Z 4348a3207684ec0ef401a834e1562981 diff --git a/manifest.uuid b/manifest.uuid index 48ec1465b4..611c0d9d53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4afdd5ae53ef0ff7c0fde74eaa04638c923c679b \ No newline at end of file +0bacb879e18026f2a8e22fe3e4bc8d27de5c4416 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2f3dc77e12..5471f71dcf 100644 --- a/src/where.c +++ b/src/where.c @@ -686,7 +686,10 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; diff --git a/test/vtab1.test b/test/vtab1.test index f056b2f368..38aec09eae 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix vtab1 ifcapable !vtab||!schema_pragmas { finish_test @@ -43,6 +44,9 @@ ifcapable !vtab||!schema_pragmas { # # vtab1-14.*: Test 'IN' constraints - i.e. "SELECT * FROM t1 WHERE id IN(...)" # +# vtab1-18.*: Check that the LIKE optimization is not applied when the lhs +# is a virtual table column. +# #---------------------------------------------------------------------- @@ -1218,5 +1222,57 @@ do_test vtab1-17.1 { } } {} +#------------------------------------------------------------------------- +# The following tests - vtab1-18.* - test that the optimization of LIKE +# constraints in where.c plays well with virtual tables. +# +# 18.1.*: Case-insensitive LIKE. +# 18.2.*: Case-sensitive LIKE. +# unset -nocomplain echo_module_begin_fail + +do_execsql_test 18.1.0 { + CREATE TABLE t6(a, b TEXT); + CREATE INDEX i6 ON t6(b, a); + INSERT INTO t6 VALUES(1, 'Peter'); + INSERT INTO t6 VALUES(2, 'Andrew'); + INSERT INTO t6 VALUES(3, 'James'); + INSERT INTO t6 VALUES(4, 'John'); + INSERT INTO t6 VALUES(5, 'Phillip'); + INSERT INTO t6 VALUES(6, 'Bartholomew'); + CREATE VIRTUAL TABLE e6 USING echo(t6); +} + +foreach {tn sql res filter} { + 1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5} + {xFilter {SELECT rowid, * FROM 't6' WHERE b > ?} James} + + 1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4} + {xFilter {SELECT rowid, * FROM 't6' WHERE b >= ? AND b < ?} J K} + + 1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} + + 1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} +} { + set echo_module {} + do_execsql_test 18.$tn.1 $sql $res + do_test 18.$tn.2 { lrange $::echo_module 2 end } $filter +} + +do_execsql_test 18.2.0 { PRAGMA case_sensitive_like = ON } +foreach {tn sql res filter} { + 2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} + + 2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {} + {xFilter {SELECT rowid, * FROM 't6'}} +} { + set echo_module {} + do_execsql_test 18.$tn.1 $sql $res + do_test 18.$tn.2 { lrange $::echo_module 2 end } $filter +} +do_execsql_test 18.2.x { PRAGMA case_sensitive_like = OFF } + finish_test From 4e245a4c35fd2e34923f1cf047cbd74f5b67273e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 00:00:36 +0000 Subject: [PATCH 62/68] Fix compiler warnings on GCC and MSVC and fix a C89-ism that broke the build for MSVC. FossilOrigin-Name: b451c0f97f0abe78ebe6c62ff489ec1ad8a1f767 --- ext/fts3/fts3.c | 4 ++-- ext/fts3/fts3_write.c | 9 +++++---- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 6 +++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e18f2b10ec..f9aef774aa 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3139,7 +3139,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** as it would also be required by a crisis-merge that used the same input ** segments. */ - const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); @@ -3152,7 +3152,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); } sqlite3Fts3SegmentsClose(p); return rc; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index c0f25dd431..fa84357e4d 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3986,7 +3986,7 @@ static int fts3IncrmergeLoad( int nHeight = (int)aRoot[0]; NodeWriter *pNode; - pWriter->nLeafEst = ((iEnd - iStart) + 1) / FTS_MAX_APPENDABLE_HEIGHT; + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; pWriter->iStart = iStart; pWriter->iEnd = iEnd; pWriter->iAbsLevel = iAbsLevel; @@ -4005,8 +4005,8 @@ static int fts3IncrmergeLoad( } for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ - pNode = &pWriter->aNodeWriter[i]; NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); @@ -4102,7 +4102,7 @@ static int fts3IncrmergeWriter( ){ int rc; /* Return Code */ int i; /* Iterator variable */ - int nLeafEst; /* Blocks allocated for leaf nodes */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ @@ -4855,7 +4855,8 @@ static u64 fts3ChecksumIndex( }else{ iPos += (iVal - 2); cksum = cksum ^ fts3ChecksumEntry( - csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, iCol, iPos + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos ); } } diff --git a/manifest b/manifest index eb8db26a7f..07dea1242d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfts4-incr-merge\swith\strunk. -D 2012-03-29T15:11:32.438 +C Fix\scompiler\swarnings\son\sGCC\sand\sMSVC\sand\sfix\sa\sC89-ism\sthat\nbroke\sthe\sbuild\sfor\sMSVC. +D 2012-03-30T00:00:36.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f41f52a24f1a9d7f94291a0e17027e0c28e4f54b +F ext/fts3/fts3.c 111626ce72b0df93f509ebd14ce31804fed24be0 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 5fd2ec4e47faf17bf4a508d6b8ec5fc0f2c80bff F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c ceb65d6a85f44c7dd1d96f12d04e20f75884bfe3 +F ext/fts3/fts3_write.c 545c3e2add64c27b2b03f9c79619ac5e47043252 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -135,7 +135,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e -F src/expr.c 1c351f385950a40b0283328eb925641a3aa50a84 +F src/expr.c 7e40ea9f6899e31134be3c1b88b8347cf9ec40d7 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 F src/func.c c6b3c94320253a35bda43fb69cc292618e3285d6 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 0bacb879e18026f2a8e22fe3e4bc8d27de5c4416 66c4aaadda433c9d479a25c27cdff84560088402 -R 44c1548cf550b784321f894b0b9c8ec7 -U dan -Z 991a128a11b720bb56ac07d77a5267e5 +P 4d6de3e9bef3487f2d89167939ab2c42872d05b3 +R 6a2a98746d07ae9d5d747d484bebf83b +U drh +Z 63420a987b6627df7e88c714721af80a diff --git a/manifest.uuid b/manifest.uuid index 035ce476f1..a99ae83262 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d6de3e9bef3487f2d89167939ab2c42872d05b3 \ No newline at end of file +b451c0f97f0abe78ebe6c62ff489ec1ad8a1f767 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1915454742..5e3f1204a1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2599,11 +2599,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** loading. */ if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ - u8 op; + u8 exprOp; assert( nFarg==1 ); assert( pFarg->a[0].pExpr!=0 ); - op = pFarg->a[0].pExpr->op; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); testcase( pDef->flags==SQLITE_FUNC_LENGTH ); From e1da8fadcc2ded2a17f3675a7837b55b4bd52c0b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 00:05:57 +0000 Subject: [PATCH 63/68] In the ".output" command-line shell, if the first character of the output filename is '|' then use popen() instead of fopen(). FossilOrigin-Name: fa82062c659ffbe7ad01106d3ef54d7bb44f1f44 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 17 ++++++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 07dea1242d..80e5365422 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\son\sGCC\sand\sMSVC\sand\sfix\sa\sC89-ism\sthat\nbroke\sthe\sbuild\sfor\sMSVC. -D 2012-03-30T00:00:36.020 +C In\sthe\s".output"\scommand-line\sshell,\sif\sthe\sfirst\scharacter\sof\sthe\soutput\nfilename\sis\s'|'\sthen\suse\spopen()\sinstead\sof\sfopen(). +D 2012-03-30T00:05:57.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c f6f141cb1ea13f1e6564d3e162700e4937baa2a1 -F src/shell.c 3179db5d4ff33d62d59a024dbfd2a116390ef7b0 +F src/shell.c abf18d6ee54f2631860a98fdd7ab1327f470db67 F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h 3756ece33f1e7f8fe2adf8e523566825c809316e @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 4d6de3e9bef3487f2d89167939ab2c42872d05b3 -R 6a2a98746d07ae9d5d747d484bebf83b +P b451c0f97f0abe78ebe6c62ff489ec1ad8a1f767 +R c9f2c82ab656ee1565b2ea7a350213e8 U drh -Z 63420a987b6627df7e88c714721af80a +Z 500f8e52607292e3d3c61e303b3bc0be diff --git a/manifest.uuid b/manifest.uuid index a99ae83262..f5e31b5289 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b451c0f97f0abe78ebe6c62ff489ec1ad8a1f767 \ No newline at end of file +fa82062c659ffbe7ad01106d3ef54d7bb44f1f44 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 52b5c4227b..4287ef17d8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -68,6 +68,8 @@ # include #define isatty(h) _isatty(h) #define access(f,m) _access((f),(m)) +#define popen(a,b) _popen((a),(b)) +#define pclose(x) _pclose(x) #else /* Make sure isatty() has a prototype. */ @@ -1999,11 +2001,24 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ if( p->out!=stdout ){ - fclose(p->out); + if( p->outfile[0]=='|' ){ + pclose(p->out); + }else{ + fclose(p->out); + } } if( strcmp(azArg[1],"stdout")==0 ){ p->out = stdout; sqlite3_snprintf(sizeof(p->outfile), p->outfile, "stdout"); + }else if( azArg[1][0]=='|' ){ + p->out = popen(&azArg[1][1], "w"); + if( p->out==0 ){ + fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]); + p->out = stdout; + rc = 1; + }else{ + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); + } }else{ p->out = fopen(azArg[1], "wb"); if( p->out==0 ){ From f9df44980925d2afd08cf33343613d61655f545d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 12:10:38 +0000 Subject: [PATCH 64/68] Change the name of a local variable from "not" to "bNot" to lessen the chances of it colliding with some prior #define in the appliation. FossilOrigin-Name: cbdd86387630600b309de4aaeaa131ec7b053ce2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/parse.y | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 80e5365422..29878944a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".output"\scommand-line\sshell,\sif\sthe\sfirst\scharacter\sof\sthe\soutput\nfilename\sis\s'|'\sthen\suse\spopen()\sinstead\sof\sfopen(). -D 2012-03-30T00:05:57.960 +C Change\sthe\sname\sof\sa\slocal\svariable\sfrom\s"not"\sto\s"bNot"\sto\slessen\sthe\nchances\sof\sit\scolliding\swith\ssome\sprior\s#define\sin\sthe\sappliation. +D 2012-03-30T12:10:38.151 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 85988507fa20acc60defb834722eddf4633e4aeb F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 -F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e +F src/parse.y 537c8db136af5f481630becdc0c8bdd36a704c30 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P b451c0f97f0abe78ebe6c62ff489ec1ad8a1f767 -R c9f2c82ab656ee1565b2ea7a350213e8 +P fa82062c659ffbe7ad01106d3ef54d7bb44f1f44 +R e9f2b9678dec950221629d45134c94ac U drh -Z 500f8e52607292e3d3c61e303b3bc0be +Z d33972f4357f472f3903792c8f2dee06 diff --git a/manifest.uuid b/manifest.uuid index f5e31b5289..ee70dc167e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa82062c659ffbe7ad01106d3ef54d7bb44f1f44 \ No newline at end of file +cbdd86387630600b309de4aaeaa131ec7b053ce2 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index ed18e7f973..fb9c4fdbec 100644 --- a/src/parse.y +++ b/src/parse.y @@ -75,7 +75,7 @@ struct LimitVal { */ struct LikeOp { Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ + int bNot; /* True if the NOT keyword is present */ }; /* @@ -881,16 +881,16 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} +likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -901,7 +901,7 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; From ccdf2025b5f50c8b734c372f71f88f246e1087bd Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 13:34:17 +0000 Subject: [PATCH 65/68] Fix an FTS4 test script problem for windows. FossilOrigin-Name: 36aa6665e709b5942b3558afbd555058b42f2c78 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts4merge3.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 29878944a9..0881a2a23e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sa\slocal\svariable\sfrom\s"not"\sto\s"bNot"\sto\slessen\sthe\nchances\sof\sit\scolliding\swith\ssome\sprior\s#define\sin\sthe\sappliation. -D 2012-03-30T12:10:38.151 +C Fix\san\sFTS4\stest\sscript\sproblem\sfor\swindows. +D 2012-03-30T13:34:17.656 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -502,7 +502,7 @@ F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 -F test/fts4merge3.test 7ca2225996f6c4e12efbd4fe34ce8a805adecfb2 +F test/fts4merge3.test 640611c05f01e9e018ac10afd188b018d8fcd4e5 F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P fa82062c659ffbe7ad01106d3ef54d7bb44f1f44 -R e9f2b9678dec950221629d45134c94ac +P cbdd86387630600b309de4aaeaa131ec7b053ce2 +R 88f6eca232fcd821cd715bbadb803c37 U drh -Z d33972f4357f472f3903792c8f2dee06 +Z a41a68a88bffc936799088c08283176f diff --git a/manifest.uuid b/manifest.uuid index ee70dc167e..03525e8dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbdd86387630600b309de4aaeaa131ec7b053ce2 \ No newline at end of file +36aa6665e709b5942b3558afbd555058b42f2c78 \ No newline at end of file diff --git a/test/fts4merge3.test b/test/fts4merge3.test index 86b44e5eee..d591329fb0 100644 --- a/test/fts4merge3.test +++ b/test/fts4merge3.test @@ -25,6 +25,7 @@ if {"" == [bc_find_binaries backcompat.test]} { return } +db close do_all_bc_test { sql2 { PRAGMA page_size = 512 } From 7da5fcb0b7b01067fe6a8d35dfd9bf7ad3307ed7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 14:59:43 +0000 Subject: [PATCH 66/68] Fix MSVC compiler warnings in test code. FossilOrigin-Name: cb7a850439c9a4a7887650d6b81d95ab8025de5b --- ext/fts3/fts3_term.c | 10 +++++++--- ext/fts3/fts3_test.c | 6 ++++++ manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- src/test1.c | 7 ++++--- src/test3.c | 2 +- src/test6.c | 16 ++++++++-------- src/test_journal.c | 10 +++++----- src/test_multiplex.c | 4 ++-- src/test_onefile.c | 20 ++++++++++---------- src/test_osinst.c | 8 ++++---- src/test_quota.c | 2 +- src/test_stat.c | 9 +++++---- src/test_wholenumber.c | 4 ++-- 14 files changed, 73 insertions(+), 61 deletions(-) diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index 2108fc1251..f4215baf51 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -73,6 +73,7 @@ static int fts3termConnectMethod( Fts3termTable *p; /* Virtual table object to return */ int iIndex = 0; + UNUSED_PARAMETER(pCtx); if( argc==5 ){ iIndex = atoi(argv[4]); argc--; @@ -231,12 +232,12 @@ static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ if( v==1 ){ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); - pCsr->iCol += v; + pCsr->iCol += (int)v; pCsr->iPos = 0; pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); } - pCsr->iPos += (v - 2); + pCsr->iPos += (int)(v - 2); return SQLITE_OK; } @@ -357,7 +358,10 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ - 0 /* xRename */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 01873d4978..0fe63c1c75 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -161,6 +161,8 @@ static int fts3_near_match_cmd( Tcl_Obj **apExprToken; int nExprToken; + UNUSED_PARAMETER(clientData); + /* Must have 3 or more arguments. */ if( objc<3 || (objc%2)==0 ){ Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?..."); @@ -314,6 +316,7 @@ static int fts3_configure_incr_load_cmd( Tcl_SetObjResult(interp, pRet); Tcl_DecrRefCount(pRet); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } @@ -352,6 +355,8 @@ static int testTokenizerCreate( sqlite3_tokenizer **ppTokenizer ){ test_tokenizer *pNew; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); pNew = sqlite3_malloc(sizeof(test_tokenizer)); if( !pNew ) return SQLITE_NOMEM; @@ -507,6 +512,7 @@ static int fts3_test_tokenizer_cmd( (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) )); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } diff --git a/manifest b/manifest index 0881a2a23e..0c8e86ab0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sFTS4\stest\sscript\sproblem\sfor\swindows. -D 2012-03-30T13:34:17.656 +C Fix\sMSVC\scompiler\swarnings\sin\stest\scode. +D 2012-03-30T14:59:43.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -73,8 +73,8 @@ F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 51a3a34c217e24678a133782c1dfb6f2f70fe559 -F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6 -F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 +F ext/fts3/fts3_term.c 41e82ad335213d1c24356cf310dca1d3c13e7366 +F ext/fts3/fts3_test.c f3ef8ae1b802383c4d24fd70774cb87d52841d5f F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -190,12 +190,12 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 086dfdd72e5892de223968a258e1ccbd9693e717 -F src/test1.c 07f30e8714bab94d8de8a88865d9c59bc512a1a8 +F src/test1.c 8631f728c9c3afcf79e3e22b510e451435e3d15e F src/test2.c 711555927f1f7e8db9aab86b512bc6934a774abe -F src/test3.c 91d3f1a09cfae3533ef17d8b484a160f3d1f1a21 +F src/test3.c f82399ec50d9cd7378bf9d6db6c1409d5e77b042 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 -F src/test6.c 846ed1ed2f470de9b1e205fe3878a12e237b3e19 +F src/test6.c 3329df2dbc0293031d763947ec08c9eef982ef1d F src/test7.c 2e0781754905c8adc3268d8f0967e7633af58843 F src/test8.c 61b41d79509a479dec1ac32b6d4209b27c4b1ba5 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -212,28 +212,28 @@ F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 +F src/test_journal.c b964473ff1b7a65626763f068fa6a810385d1fbf F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 3f5903a1528fd32fe4c472a3bd0259128d8faaef -F src/test_multiplex.c 0404a61d7795438be5ee5fd3711eed80724df34d +F src/test_multiplex.c 30ca0348953abd3add46fe4ee19e3f9e669b7e56 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e -F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec -F src/test_osinst.c 6abf0a37ce831120c4ef1b913afdd813e7ac1a73 +F src/test_onefile.c 5e1382e7844c703c77c4c2aee82f8359555b5a8e +F src/test_osinst.c 7f790ac89c5a585d51b341274d9691c3391e0923 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c b4a6519417d87870e7ef5838dbf3cae164dcc28d +F src/test_quota.c a545115f837da4ef32f6b5578f147b44cfb13fd7 F src/test_quota.h 9ffa1d3ad6d0a6a24e8670ea64b909c717ec3358 F src/test_rtree.c 6d06306e29946dc36f528a3a2cdc3add794656f1 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_stat.c 80271ad7d776a79babe0e025bb3a1bfcd3a3cfb1 +F src/test_stat.c d7035cfcc0ff1f93c000b621f36524318e004e11 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c 73f46bd9b5183ebcb77da22773886b81157cdc3d F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 -F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 +F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cbdd86387630600b309de4aaeaa131ec7b053ce2 -R 88f6eca232fcd821cd715bbadb803c37 +P 36aa6665e709b5942b3558afbd555058b42f2c78 +R 19f0f46e7579b91721fe0a073b822b78 U drh -Z a41a68a88bffc936799088c08283176f +Z 3f9228d0d5429417687badcbcda00070 diff --git a/manifest.uuid b/manifest.uuid index 03525e8dd3..012c2f19a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36aa6665e709b5942b3558afbd555058b42f2c78 \ No newline at end of file +cb7a850439c9a4a7887650d6b81d95ab8025de5b \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index e1220cf11f..eed6474b90 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3263,7 +3263,7 @@ static int test_bind_text16( char *value; int rc; - void (*xDel)() = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); + void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); Tcl_Obj *oStmt = objv[objc-4]; Tcl_Obj *oN = objv[objc-3]; Tcl_Obj *oString = objv[objc-2]; @@ -3611,7 +3611,7 @@ static int test_prepare( if( bytes>=0 ){ bytes = bytes - (zTail-zSql); } - if( strlen(zTail) /* ** The background thread that does file locking. */ @@ -6128,7 +6129,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA {"sqlite3_column_database_name16", - test_stmt_utf16, sqlite3_column_database_name16}, + test_stmt_utf16, (void*)sqlite3_column_database_name16}, {"sqlite3_column_table_name16", test_stmt_utf16, (void*)sqlite3_column_table_name16}, {"sqlite3_column_origin_name16", test_stmt_utf16, (void*)sqlite3_column_origin_name16}, #endif diff --git a/src/test3.c b/src/test3.c index 9bac2cac13..947ded7d91 100644 --- a/src/test3.c +++ b/src/test3.c @@ -465,7 +465,7 @@ static int btree_varint_test( if( Tcl_GetInt(interp, argv[4], (int*)&incr) ) return TCL_ERROR; in = start; in *= mult; - for(i=0; i9 || n1<1 ){ diff --git a/src/test6.c b/src/test6.c index 5f64cacca0..e2cee09193 100644 --- a/src/test6.c +++ b/src/test6.c @@ -177,7 +177,7 @@ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ iSkip = 512; } if( (iAmt-iSkip)>0 ){ - rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], iAmt-iSkip, iOff+iSkip); + rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], (int)(iAmt-iSkip), iOff+iSkip); } return rc; } @@ -306,8 +306,8 @@ static int writeListSync(CrashFile *pFile, int isCrash){ } case 3: { /* Trash sectors */ u8 *zGarbage; - int iFirst = (pWrite->iOffset/g.iSectorSize); - int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize; + int iFirst = (int)(pWrite->iOffset/g.iSectorSize); + int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); @@ -430,7 +430,7 @@ static int cfWrite( ){ CrashFile *pCrash = (CrashFile *)pFile; if( iAmt+iOfst>pCrash->iSize ){ - pCrash->iSize = iAmt+iOfst; + pCrash->iSize = (int)(iAmt+iOfst); } while( pCrash->iSize>pCrash->nData ){ u8 *zNew; @@ -454,7 +454,7 @@ static int cfTruncate(sqlite3_file *pFile, sqlite_int64 size){ CrashFile *pCrash = (CrashFile *)pFile; assert(size>=0); if( pCrash->iSize>size ){ - pCrash->iSize = size; + pCrash->iSize = (int)size; } return writeListAppend(pFile, size, 0, 0); } @@ -518,7 +518,7 @@ static int cfFileControl(sqlite3_file *pFile, int op, void *pArg){ i64 nByte = *(i64 *)pArg; if( nByte>pCrash->iSize ){ if( SQLITE_OK==writeListAppend(pFile, nByte, 0, 0) ){ - pCrash->iSize = nByte; + pCrash->iSize = (int)nByte; } } return SQLITE_OK; @@ -635,11 +635,11 @@ static int cfOpen( iChunk = PENDING_BYTE; } memset(pWrapper->zData, 0, pWrapper->nData); - rc = sqlite3OsRead(pReal, pWrapper->zData, iChunk, 0); + rc = sqlite3OsRead(pReal, pWrapper->zData, (int)iChunk, 0); if( SQLITE_OK==rc && pWrapper->iSize>(PENDING_BYTE+512) && isDb ){ i64 iOff = PENDING_BYTE+512; iChunk = pWrapper->iSize - iOff; - rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], iChunk, iOff); + rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], (int)iChunk, iOff); } }else{ rc = SQLITE_NOMEM; diff --git a/src/test_journal.c b/src/test_journal.c index 00567db598..4cb51fbcf5 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -404,7 +404,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ /* Calculate and store a checksum for each page in the database file. */ if( rc==SQLITE_OK ){ int ii; - for(ii=0; rc==SQLITE_OK && iinPage; ii++){ + for(ii=0; rc==SQLITE_OK && ii<(int)pMain->nPage; ii++){ i64 iOff = (i64)(pMain->nPagesize) * (i64)ii; if( iOff==PENDING_BYTE ) continue; rc = sqlite3OsRead(pMain->pReal, aData, pMain->nPagesize, iOff); @@ -466,7 +466,7 @@ static int readJournalFile(jt_file *p, jt_file *pMain){ continue; } } - nRec = (iSize-iOff) / (pMain->nPagesize+8); + nRec = (u32)((iSize-iOff) / (pMain->nPagesize+8)); } /* Read all the records that follow the journal-header just read. */ @@ -538,7 +538,7 @@ static int jtWrite( } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ - if( iAmtnPagesize + if( iAmt<(int)p->nPagesize && p->nPagesize%iAmt==0 && iOfst>=(PENDING_BYTE+512) && iOfst+iAmt<=PENDING_BYTE+p->nPagesize @@ -549,7 +549,7 @@ static int jtWrite( ** pending-byte page. */ }else{ - u32 pgno = iOfst/p->nPagesize + 1; + u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); @@ -578,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ u32 pgno; u32 locking_page = (u32)(PENDING_BYTE/p->nPagesize+1); - for(pgno=size/p->nPagesize+1; pgno<=p->nPage; pgno++){ + for(pgno=(u32)(size/p->nPagesize+1); pgno<=p->nPage; pgno++){ assert( pgno==locking_page || sqlite3BitvecTest(p->pWritable, pgno) ); } } diff --git a/src/test_multiplex.c b/src/test_multiplex.c index e2b6720652..32dd8e33e8 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -529,7 +529,7 @@ static int multiplexOpen( pGroup->bEnabled = -1; pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", (flags & SQLITE_OPEN_MAIN_DB)==0); - pGroup->szChunk = sqlite3_uri_int64(zUri, "chunksize", + pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize", SQLITE_MULTIPLEX_CHUNK_SIZE); pGroup->szChunk = (pGroup->szChunk+0xffff)&~0xffff; if( zName ){ @@ -597,7 +597,7 @@ static int multiplexOpen( bExists = multiplexSubSize(pGroup, 1, &rc)>0; if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ - pGroup->szChunk = sz; + pGroup->szChunk = (int)sz; }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ pGroup->bEnabled = 0; } diff --git a/src/test_onefile.c b/src/test_onefile.c index cd7db008cf..adc0d13d9f 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -288,7 +288,7 @@ static int tmpWrite( ){ tmp_file *pTmp = (tmp_file *)pFile; if( (iAmt+iOfst)>pTmp->nAlloc ){ - int nNew = 2*(iAmt+iOfst+pTmp->nAlloc); + int nNew = (int)(2*(iAmt+iOfst+pTmp->nAlloc)); char *zNew = sqlite3_realloc(pTmp->zAlloc, nNew); if( !zNew ){ return SQLITE_NOMEM; @@ -297,7 +297,7 @@ static int tmpWrite( pTmp->nAlloc = nNew; } memcpy(&pTmp->zAlloc[iOfst], zBuf, iAmt); - pTmp->nSize = MAX(pTmp->nSize, iOfst+iAmt); + pTmp->nSize = (int)MAX(pTmp->nSize, iOfst+iAmt); return SQLITE_OK; } @@ -306,7 +306,7 @@ static int tmpWrite( */ static int tmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ tmp_file *pTmp = (tmp_file *)pFile; - pTmp->nSize = MIN(pTmp->nSize, size); + pTmp->nSize = (int)MIN(pTmp->nSize, size); return SQLITE_OK; } @@ -418,7 +418,7 @@ static int fsRead( /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -453,14 +453,14 @@ static int fsWrite( }else{ rc = pF->pMethods->xWrite(pF, zBuf, iAmt, iOfst+BLOCKSIZE); if( rc==SQLITE_OK ){ - pReal->nDatabase = MAX(pReal->nDatabase, iAmt+iOfst); + pReal->nDatabase = (int)MAX(pReal->nDatabase, iAmt+iOfst); } } }else{ /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -475,7 +475,7 @@ static int fsWrite( } } if( rc==SQLITE_OK ){ - pReal->nJournal = MAX(pReal->nJournal, iAmt+iOfst); + pReal->nJournal = (int)MAX(pReal->nJournal, iAmt+iOfst); } } @@ -489,9 +489,9 @@ static int fsTruncate(sqlite3_file *pFile, sqlite_int64 size){ fs_file *p = (fs_file *)pFile; fs_real_file *pReal = p->pReal; if( p->eType==DATABASE_FILE ){ - pReal->nDatabase = MIN(pReal->nDatabase, size); + pReal->nDatabase = (int)MIN(pReal->nDatabase, size); }else{ - pReal->nJournal = MIN(pReal->nJournal, size); + pReal->nJournal = (int)MIN(pReal->nJournal, size); } return SQLITE_OK; } @@ -641,7 +641,7 @@ static int fsOpen( pReal->nBlob = BLOBSIZE; }else{ unsigned char zS[4]; - pReal->nBlob = size; + pReal->nBlob = (int)size; rc = pRealFile->pMethods->xRead(pRealFile, zS, 4, 0); pReal->nDatabase = (zS[0]<<24)+(zS[1]<<16)+(zS[2]<<8)+zS[3]; if( rc==SQLITE_OK ){ diff --git a/src/test_osinst.c b/src/test_osinst.c index b7f350577c..35e168c418 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -242,7 +242,7 @@ static sqlite3_uint64 vfslog_time(){ } #endif -static void vfslog_call(sqlite3_vfs *, int, int, int, int, int, int); +static void vfslog_call(sqlite3_vfs *, int, int, sqlite3_int64, int, int, int); static void vfslog_string(sqlite3_vfs *, const char *); /* @@ -648,7 +648,7 @@ static void vfslog_call( sqlite3_vfs *pVfs, int eEvent, int iFileid, - int nClick, + sqlite3_int64 nClick, int return_code, int size, int offset @@ -661,7 +661,7 @@ static void vfslog_call( zRec = (unsigned char *)&p->aBuf[p->nBuf]; put32bits(&zRec[0], eEvent); put32bits(&zRec[4], iFileid); - put32bits(&zRec[8], nClick); + put32bits(&zRec[8], (unsigned int)(nClick&0xffff)); put32bits(&zRec[12], return_code); put32bits(&zRec[16], size); put32bits(&zRec[20], offset); @@ -1043,7 +1043,7 @@ static int vlogColumn( } case 1: { char *zStr = pCsr->zTransient; - if( val!=0 && valnFile ){ + if( val!=0 && val<(unsigned)pCsr->nFile ){ zStr = pCsr->azFile[val]; } sqlite3_result_text(ctx, zStr, -1, SQLITE_TRANSIENT); diff --git a/src/test_quota.c b/src/test_quota.c index c8ed7389e5..e749851238 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -1060,7 +1060,7 @@ size_t sqlite3_quota_fwrite( } if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){ iEnd = pGroup->iLimit - pGroup->iSize + pFile->iSize; - nmemb = (iEnd - iOfst)/size; + nmemb = (size_t)((iEnd - iOfst)/size); iEnd = iOfst + size*nmemb; szNew = pGroup->iSize - pFile->iSize + iEnd; } diff --git a/src/test_stat.c b/src/test_stat.c index a0893b3974..e83cebebe0 100644 --- a/src/test_stat.c +++ b/src/test_stat.c @@ -324,12 +324,13 @@ static int statDecodePage(Btree *pBt, StatPage *p){ u64 dummy; iOff += sqlite3GetVarint(&aData[iOff], &dummy); } - if( nPayload>p->nMxPayload ) p->nMxPayload = nPayload; + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; getLocalPayload(nUsable, p->flags, nPayload, &nLocal); pCell->nLocal = nLocal; + assert( nLocal>=0 ); assert( nPayload>=nLocal ); assert( nLocal<=(nUsable-35) ); - if( nPayload>nLocal ){ + if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); @@ -378,7 +379,7 @@ static void statSizeAndOffset(StatCursor *pCsr){ x[0] = pCsr->iPageno; if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ pCsr->iOffset = x[0]; - pCsr->szPage = x[1]; + pCsr->szPage = (int)x[1]; } } @@ -400,7 +401,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ int nPage; - u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); sqlite3PagerPagecount(pPager, &nPage); if( nPage==0 ){ pCsr->isEof = 1; diff --git a/src/test_wholenumber.c b/src/test_wholenumber.c index 150dc95ac4..7c42d01691 100644 --- a/src/test_wholenumber.c +++ b/src/test_wholenumber.c @@ -33,8 +33,8 @@ typedef struct wholenumber_cursor wholenumber_cursor; struct wholenumber_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - unsigned iValue; /* Current value */ - unsigned mxValue; /* Maximum value */ + sqlite3_int64 iValue; /* Current value */ + sqlite3_int64 mxValue; /* Maximum value */ }; /* Methods for the wholenumber module */ From f0146403a62fdabac7acca4b9d9ec58b99213efc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Mar 2012 15:57:45 +0000 Subject: [PATCH 67/68] Simplify the winRead and winWrite VFS functions to reduce the number of system calls. FossilOrigin-Name: b34491869c4fb31d2fdd14c94a7db2e1c0e572ba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 33 ++++++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 4181c0f6f2..47f54b8cc9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\snamed\sCHECK\sconstraint\sfails,\sinclude\sthe\sname\sof\sthe\sconstraint\nin\sthe\serror\smessage. -D 2012-03-30T15:48:48.787 +C Simplify\sthe\swinRead\sand\swinWrite\sVFS\sfunctions\sto\sreduce\sthe\snumber\nof\ssystem\scalls. +D 2012-03-30T15:57:45.451 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 -F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 +F src/os_win.c c7b26b78c5e62de7355ad79e0ce79d7b0f5fccc4 F src/pager.c 85988507fa20acc60defb834722eddf4633e4aeb F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y eb054bb40a5bf90d3422a01ed0e5df229461727a @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cb7a850439c9a4a7887650d6b81d95ab8025de5b 9a0f90d9deb335ac71044b8afa81538d85cc7ccf -R 3383713cde4154ba8c7927e87d965912 +P 1b75f301affac654bee24fa247046ea0782d3c4d +R 2cc6bffd476fbc89b05bef18789bd072 U drh -Z 044e1c348f17a4e06833c51581164485 +Z 6ac97e4f60396b50617248087122bd36 diff --git a/manifest.uuid b/manifest.uuid index a240625a11..8107006afc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b75f301affac654bee24fa247046ea0782d3c4d \ No newline at end of file +b34491869c4fb31d2fdd14c94a7db2e1c0e572ba \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8b86c7ed59..5826be688f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1616,6 +1616,9 @@ static int winClose(sqlite3_file *id){ } #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); + if( rc ){ + pFile->h = NULL; + } OpenCounter(-1); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), @@ -1633,6 +1636,7 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ + OVERLAPPED overlapped; /* The offset for ReadFile. */ winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ int nRetry = 0; /* Number of retrys */ @@ -1641,10 +1645,11 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); - if( seekWinFile(pFile, offset) ){ - return SQLITE_FULL; - } - while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; @@ -1671,7 +1676,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc; /* True if error has occured, else false */ + int rc = 0; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -1682,19 +1687,29 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - rc = seekWinFile(pFile, offset); - if( rc==0 ){ + { + OVERLAPPED overlapped; /* The offset for WriteFile. */ u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( nRem>0 ){ - if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } - if( nWrite<=0 ) break; + if( nWrite<=0 ){ + lastErrno = osGetLastError(); + break; + } + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); aRem += nWrite; nRem -= nWrite; } From 00fa55d7da3ff31b5c354884b8e507f50339b756 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 30 Mar 2012 16:44:33 +0000 Subject: [PATCH 68/68] Avoid using the OVERLAPPED struct on WinCE. FossilOrigin-Name: 196ca3a8b007b9f792e969893d981f6c5aa2fccc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 47f54b8cc9..17608576f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\swinRead\sand\swinWrite\sVFS\sfunctions\sto\sreduce\sthe\snumber\nof\ssystem\scalls. -D 2012-03-30T15:57:45.451 +C Avoid\susing\sthe\sOVERLAPPED\sstruct\son\sWinCE. +D 2012-03-30T16:44:33.303 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 -F src/os_win.c c7b26b78c5e62de7355ad79e0ce79d7b0f5fccc4 +F src/os_win.c 5e9e933a412ab35de2a6506b3c6a8295b31b309e F src/pager.c 85988507fa20acc60defb834722eddf4633e4aeb F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y eb054bb40a5bf90d3422a01ed0e5df229461727a @@ -999,7 +999,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1b75f301affac654bee24fa247046ea0782d3c4d -R 2cc6bffd476fbc89b05bef18789bd072 -U drh -Z 6ac97e4f60396b50617248087122bd36 +P b34491869c4fb31d2fdd14c94a7db2e1c0e572ba +R 7c7c91e4f03aba786b8f3e6627280744 +U mistachkin +Z 7d31e5badb9fd33ec4e58af382112a75 diff --git a/manifest.uuid b/manifest.uuid index 8107006afc..aa0af4ede5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b34491869c4fb31d2fdd14c94a7db2e1c0e572ba \ No newline at end of file +196ca3a8b007b9f792e969893d981f6c5aa2fccc \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5826be688f..358b70c128 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1636,7 +1636,9 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ +#if !SQLITE_OS_WINCE OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ int nRetry = 0; /* Number of retrys */ @@ -1645,11 +1647,18 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE + if( seekWinFile(pFile, offset) ){ + return SQLITE_FULL; + } + while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; @@ -1687,19 +1696,32 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE + rc = seekWinFile(pFile, offset); + if( rc==0 ){ +#else { +#endif +#if !SQLITE_OS_WINCE OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ +#if !SQLITE_OS_WINCE memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif while( nRem>0 ){ +#if SQLITE_OS_WINCE + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } @@ -1707,9 +1729,11 @@ static int winWrite( lastErrno = osGetLastError(); break; } +#if !SQLITE_OS_WINCE offset += nWrite; overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif aRem += nWrite; nRem -= nWrite; }