diff --git a/VERSION b/VERSION index 584a914c29..f7e5aa84c2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.11 +3.7.12 diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 96905fc14b..f9aef774aa 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -70,7 +70,7 @@ ** A doclist is stored like this: ** ** array { -** varint docid; +** varint docid; (delta from previous doclist) ** array { (position list for column 0) ** varint position; (2 more than the delta from previous position) ** } @@ -101,8 +101,8 @@ ** at D signals the start of a new column; the 1 at E indicates that the ** new column is column number 1. There are two positions at 12 and 45 ** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the next docid. It has one position 72 (72-2) and then -** terminates with the 0 at K. +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. ** ** A "position-list" is the list of positions for multiple columns for ** a single docid. A "column-list" is the set of positions for a single @@ -570,6 +570,18 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ } } +/* +** Create the %_stat table if it does not already exist. +*/ +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done @@ -630,11 +642,9 @@ static int fts3CreateTables(Fts3Table *p){ p->zDb, p->zName ); } + assert( p->bHasStat==p->bFts4 ); if( p->bHasStat ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); + sqlite3Fts3CreateStatTable(&rc, p); } return rc; } @@ -741,7 +751,7 @@ static void fts3Appendf( static char *fts3QuoteId(char const *zInput){ int nRet; char *zRet; - nRet = 2 + strlen(zInput)*2 + 1; + nRet = 2 + (int)strlen(zInput)*2 + 1; zRet = sqlite3_malloc(nRet); if( zRet ){ int i; @@ -997,7 +1007,7 @@ static int fts3ContentColumns( nCol = sqlite3_column_count(pStmt); for(i=0; inMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bFts4 = isFts4; p->bDescIdx = bDescIdx; + p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -1327,6 +1340,16 @@ static int fts3InitVtab( rc = fts3CreateTables(p); } + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + int rc2 = SQLITE_OK; + fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", + p->zDb, p->zName); + if( rc2==SQLITE_OK ) p->bHasStat = 1; + } + /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); @@ -2330,7 +2353,7 @@ static int fts3DoclistOrMerge( } *paOut = aOut; - *pnOut = (p-aOut); + *pnOut = (int)(p-aOut); assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); return SQLITE_OK; } @@ -2394,7 +2417,7 @@ static void fts3DoclistPhraseMerge( } } - *pnRight = p - aOut; + *pnRight = (int)(p - aOut); } /* @@ -2670,7 +2693,7 @@ static int fts3SegReaderCursor( */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ - int iLangid, + int iLangid, /* Language-id to search */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2688,12 +2711,7 @@ int sqlite3Fts3SegReaderCursor( assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); assert( isPrefix==0 || isScan==0 ); - /* "isScan" is only set to true by the ft4aux module, an ordinary - ** full-text tables. */ - assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3MultiSegReader)); - return fts3SegReaderCursor( p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); @@ -2958,7 +2976,7 @@ static int fts3FilterMethod( if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -3101,8 +3119,42 @@ static int fts3UpdateMethod( ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + } + sqlite3Fts3SegmentsClose(p); return rc; } @@ -3110,13 +3162,14 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(pVtab); assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; return SQLITE_OK; } @@ -3411,11 +3464,15 @@ static int fts3RenameMethod( ** Flush the contents of the pending-terms table to disk. */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - return fts3SyncMethod(pVtab); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; } /* @@ -3775,7 +3832,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); sqlite3_free(aPoslist); aPoslist = pList; - nPoslist = aOut - aPoslist; + nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); pPhrase->doclist.pList = 0; @@ -3819,7 +3876,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aOut; if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); }else{ sqlite3_free(aOut); pPhrase->doclist.pList = 0; @@ -3915,7 +3972,7 @@ void sqlite3Fts3DoclistPrev( iMul = (bDescIdx ? -1 : 1); } - *pnList = pEnd - pNext; + *pnList = (int)(pEnd - pNext); *ppIter = pNext; *piDocid = iDocid; }else{ @@ -3929,7 +3986,7 @@ void sqlite3Fts3DoclistPrev( }else{ char *pSave = p; fts3ReversePoslist(aDoclist, &p); - *pnList = (pSave - p); + *pnList = (int)(pSave - p); } *ppIter = p; } @@ -3989,7 +4046,7 @@ static int fts3EvalPhraseNext( } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); - pDL->nList = (pIter - pDL->pList); + pDL->nList = (int)(pIter - pDL->pList); /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was @@ -4330,7 +4387,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); /* Determine which, if any, tokens in the expression should be deferred. */ - if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( @@ -4347,8 +4404,8 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Expr **ppOr = apOr; fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = pTC-aTC; - nOr = ppOr-apOr; + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); if( rc==SQLITE_OK ){ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); @@ -4420,7 +4477,7 @@ static int fts3EvalNearTrim( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 ); if( res ){ - nNew = (pOut - pPhrase->doclist.pList) - 1; + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; assert( pPhrase->doclist.pList[nNew]=='\0' ); assert( nNew<=pPhrase->doclist.nList && nNew>0 ); memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); @@ -5146,6 +5203,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } + /* ** Return SQLITE_CORRUPT_VTAB. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 393cd6aea1..7d6c34789f 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -67,6 +67,9 @@ extern const sqlite3_api_routines *sqlite3_api; #ifndef MIN # define MIN(x,y) ((x)<(y)?(x):(y)) #endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif /* ** Maximum length of a varint encoded integer. The varint format is different @@ -121,7 +124,7 @@ extern const sqlite3_api_routines *sqlite3_api; # define NEVER(X) (0) #else # define ALWAYS(x) (x) -# define NEVER(X) (x) +# define NEVER(x) (x) #endif /* @@ -131,6 +134,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* ** Macro used to suppress compiler warnings for unused parameters. @@ -193,36 +197,43 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ + u8 bAutoincrmerge; /* True if automerge=1 */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[28]; + sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* TODO: Fix the first paragraph of this comment. - ** + /* ** The following array of hash tables is used to buffer pending index - ** updates during transactions. Variable nPendingData estimates the memory - ** size of the pending data, including hash table overhead, not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, the buffer - ** is flushed automatically. Variable iPrevDocid is the docid of the most - ** recently inserted record. + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. */ int nIndex; /* Size of aIndex[] */ struct Fts3Index { @@ -421,6 +432,7 @@ int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 @@ -472,6 +484,8 @@ struct Fts3MultiSegReader { int nDoclist; /* Size of aDoclist[] in bytes */ }; +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); @@ -481,6 +495,7 @@ void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +void sqlite3Fts3CreateStatTable(int*, Fts3Table*); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 89bf3ebf6e..a2bff2e1d3 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -79,9 +79,9 @@ static int fts3auxConnectMethod( } zDb = argv[1]; - nDb = strlen(zDb); + nDb = (int)strlen(zDb); zFts3 = argv[3]; - nFts3 = strlen(zFts3); + nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index fd5bc9786b..dd3706cc80 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -794,8 +794,8 @@ static int fts3MatchinfoCheck( ){ if( (cArg==FTS3_MATCHINFO_NPHRASE) || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) || (cArg==FTS3_MATCHINFO_LCS) || (cArg==FTS3_MATCHINFO_HITS) diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index 2108fc1251..f4215baf51 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -73,6 +73,7 @@ static int fts3termConnectMethod( Fts3termTable *p; /* Virtual table object to return */ int iIndex = 0; + UNUSED_PARAMETER(pCtx); if( argc==5 ){ iIndex = atoi(argv[4]); argc--; @@ -231,12 +232,12 @@ static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ if( v==1 ){ pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); - pCsr->iCol += v; + pCsr->iCol += (int)v; pCsr->iPos = 0; pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); } - pCsr->iPos += (v - 2); + pCsr->iPos += (int)(v - 2); return SQLITE_OK; } @@ -357,7 +358,10 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ - 0 /* xRename */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 01873d4978..0fe63c1c75 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -161,6 +161,8 @@ static int fts3_near_match_cmd( Tcl_Obj **apExprToken; int nExprToken; + UNUSED_PARAMETER(clientData); + /* Must have 3 or more arguments. */ if( objc<3 || (objc%2)==0 ){ Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?..."); @@ -314,6 +316,7 @@ static int fts3_configure_incr_load_cmd( Tcl_SetObjResult(interp, pRet); Tcl_DecrRefCount(pRet); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } @@ -352,6 +355,8 @@ static int testTokenizerCreate( sqlite3_tokenizer **ppTokenizer ){ test_tokenizer *pNew; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); pNew = sqlite3_malloc(sizeof(test_tokenizer)); if( !pNew ) return SQLITE_NOMEM; @@ -507,6 +512,7 @@ static int fts3_test_tokenizer_cmd( (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) )); #endif + UNUSED_PARAMETER(clientData); return TCL_OK; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index bc20dfa3f3..fa84357e4d 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -24,6 +24,9 @@ #include #include + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + /* ** When full-text index nodes are loaded from disk, the buffer that they ** are loaded into has the following number of bytes of padding at the end @@ -63,6 +66,29 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -224,15 +250,22 @@ struct SegmentNode { #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_DOCTOTAL 22 -#define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 - #define SQL_DELETE_SEGDIR_RANGE 26 - #define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle @@ -261,9 +294,9 @@ static int fts3SqlStmt( /* 6 */ "DELETE FROM %Q.'%q_stat'", /* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " @@ -281,14 +314,61 @@ static int fts3SqlStmt( /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", /* 24 */ "", /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -326,22 +406,18 @@ static int fts3SqlStmt( return rc; } + static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ - int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ int rc; /* Return code */ - assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); - - rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); if( rc==SQLITE_OK ){ - if( eStmt==SQL_SELECT_DOCSIZE ){ - sqlite3_bind_int64(pStmt, 1, iDocid); - } + sqlite3_bind_int64(pStmt, 1, iDocid); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); @@ -360,7 +436,21 @@ int sqlite3Fts3SelectDoctotal( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); + sqlite3_stmt *pStmt = 0; + int rc; + rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)!=SQLITE_ROW + || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB + ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + } + } + *ppStmt = pStmt; + return rc; } int sqlite3Fts3SelectDocsize( @@ -368,7 +458,7 @@ int sqlite3Fts3SelectDocsize( sqlite3_int64 iDocid, /* Docid to read size data for */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); + return fts3SelectDocsize(pTab, iDocid, ppStmt); } /* @@ -458,10 +548,10 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). */ static sqlite3_int64 getAbsoluteLevel( - Fts3Table *p, - int iLangid, - int iIndex, - int iLevel + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ ){ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); @@ -472,7 +562,6 @@ static sqlite3_int64 getAbsoluteLevel( return iBase + iLevel; } - /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -1038,6 +1127,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ @@ -1085,7 +1175,7 @@ int sqlite3Fts3ReadBlock( int rc; /* Return code */ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob); + assert( pnBlob ); if( p->pSegments ){ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); @@ -1426,7 +1516,7 @@ int sqlite3Fts3MsrOvfl( int rc = SQLITE_OK; int pgsz = p->nPgsz; - assert( p->bHasStat ); + assert( p->bFts4 ); assert( pgsz>0 ); for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ @@ -1784,6 +1874,27 @@ static int fts3WriteSegment( return rc; } +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + /* ** Insert a record into the %_segdir table. */ @@ -2100,6 +2211,7 @@ static int fts3SegWriterAdd( /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: @@ -2208,6 +2320,7 @@ static int fts3SegWriterFlush( rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } @@ -2289,6 +2402,29 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + /* ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, @@ -2311,19 +2447,12 @@ static int fts3DeleteSegdir( Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ int i; /* Iterator variable */ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); for(i=0; rc==SQLITE_OK && iiStartBlock ){ - sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } + rc = fts3DeleteSegment(p, apSegment[i]); } if( rc!=SQLITE_OK ){ return rc; @@ -2899,11 +3028,28 @@ static int fts3SegmentMerge( int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + rc = sqlite3_reset(pStmt); + } + } return rc; } @@ -3014,12 +3160,13 @@ static void fts3UpdateDocTotals( return; } pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), @@ -3043,13 +3190,14 @@ static void fts3UpdateDocTotals( a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } - sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); @@ -3154,7 +3302,7 @@ static int fts3DoRebuild(Fts3Table *p){ } } } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); } sqlite3_free(aSz); @@ -3170,6 +3318,1688 @@ static int fts3DoRebuild(Fts3Table *p){ return rc; } + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + int nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert( p->iOff<=p->nNode ); + + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return nodeReaderNext(p); +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nPrefix = 0; + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + iEnd = sqlite3_column_int64(pSelect, 3); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The two +** segments to be merged are the nMin oldest segments (the ones with +** the smallest indexes) in the highest level that contains at least +** nMin segments. Multiple merges might occur in an attempt to write the +** quota of nMerge leaf blocks. +*/ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + if( rc==SQLITE_OK ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + fts3IncrmergeRelease(p, pWriter, &rc); + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->bAutoincrmerge = fts3Getint(&zParam)!=0; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc;; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrnIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken; /* Number of bytes in token */ + int iDum1, iDum2; /* Dummy variables */ + int iPos; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB; + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -3189,6 +5019,12 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoOptimize(p, 0); }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -3384,7 +5220,7 @@ static int fts3DeleteByRowid( ** tables. The schema of the virtual table being: ** ** CREATE TABLE ( -** , +** , **
HIDDEN, ** docid HIDDEN, ** HIDDEN @@ -3516,7 +5352,7 @@ int sqlite3Fts3UpdateMethod( nChng++; } - if( p->bHasStat ){ + if( p->bFts4 ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c new file mode 100644 index 0000000000..479ae9868d --- /dev/null +++ b/ext/fts3/tool/fts3view.c @@ -0,0 +1,874 @@ +/* +** This program is a debugging and analysis utility that displays +** information about an FTS3 or FTS4 index. +** +** Link this program against the SQLite3 amalgamation with the +** SQLITE_ENABLE_FTS4 compile-time option. Then run it as: +** +** fts3view DATABASE +** +** to get a list of all FTS3/4 tables in DATABASE, or do +** +** fts3view DATABASE TABLE COMMAND .... +** +** to see various aspects of the TABLE table. Type fts3view with no +** arguments for a list of available COMMANDs. +*/ +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** Extra command-line arguments: +*/ +int nExtra; +char **azExtra; + +/* +** Look for a command-line argument. +*/ +const char *findOption(const char *zName, int hasArg, const char *zDefault){ + int i; + const char *zResult = zDefault; + for(i=0; i=2000 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + if( nDoc>=200 ){ + n = 0; + pStmt = prepare(db, "SELECT count(*) FROM %s" + " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n", + n, n*100.0/nToken); + } + + nTop = atoi(findOption("top", 1, "25")); + printf("The %d most common tokens:\n", nTop); + pStmt = prepare(db, + "SELECT term, documents FROM %s" + " WHERE col='*'" + " ORDER BY documents DESC, term" + " LIMIT %d", zAux, nTop); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i++; + n = sqlite3_column_int(pStmt, 1); + printf(" %2d. %-30s %9d docs %5.2f%%\n", i, + sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc); + } + sqlite3_finalize(pStmt); + +end_vocab: + runSql(db, "ROLLBACK"); + sqlite3_free(zAux); +} + +/* +** Report on the number and sizes of segments +*/ +static void showSegmentStats(sqlite3 *db, const char *zTab){ + sqlite3_stmt *pStmt; + int nSeg = 0; + sqlite3_int64 szSeg = 0, mxSeg = 0; + int nIdx = 0; + sqlite3_int64 szIdx = 0, mxIdx = 0; + int nRoot = 0; + sqlite3_int64 szRoot = 0, mxRoot = 0; + sqlite3_int64 mx; + int nLeaf; + int n; + int pgsz; + int mxLevel; + int i; + + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nSeg = sqlite3_column_int(pStmt, 0); + szSeg = sqlite3_column_int64(pStmt, 1); + mxSeg = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(block)), max(length(block))" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block", + zTab, zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nIdx = sqlite3_column_int(pStmt, 0); + szIdx = sqlite3_column_int64(pStmt, 1); + mxIdx = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + pStmt = prepare(db, + "SELECT count(*), sum(length(root)), max(length(root))" + " FROM '%q_segdir'", + zTab); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nRoot = sqlite3_column_int(pStmt, 0); + szRoot = sqlite3_column_int64(pStmt, 1); + mxRoot = sqlite3_column_int64(pStmt, 2); + } + sqlite3_finalize(pStmt); + + printf("Number of segments....................... %9d\n", nSeg+nRoot); + printf("Number of leaf segments.................. %9d\n", nSeg-nIdx); + printf("Number of index segments................. %9d\n", nIdx); + printf("Number of root segments.................. %9d\n", nRoot); + printf("Total size of all segments............... %9lld\n", szSeg+szRoot); + printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx); + printf("Total size of all index segments......... %9lld\n", szIdx); + printf("Total size of all root segments.......... %9lld\n", szRoot); + if( nSeg>0 ){ + printf("Average size of all segments............. %11.1f\n", + (double)(szSeg+szRoot)/(double)(nSeg+nRoot)); + printf("Average size of leaf segments............ %11.1f\n", + (double)(szSeg-szIdx)/(double)(nSeg-nIdx)); + } + if( nIdx>0 ){ + printf("Average size of index segments........... %11.1f\n", + (double)szIdx/(double)nIdx); + } + if( nRoot>0 ){ + printf("Average size of root segments............ %11.1f\n", + (double)szRoot/(double)nRoot); + } + mx = mxSeg; + if( mx%d", + zTab, zTab, pgsz-45); + n = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + n = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + nLeaf = nSeg - nIdx; + printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + + pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); + mxLevel = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + + for(i=0; i<=mxLevel; i++){ + pStmt = prepare(db, + "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d)," + " count(distinct idx)" + " FROM (SELECT length(a.block) AS len, idx" + " FROM '%q_segments' a JOIN '%q_segdir' b" + " WHERE (a.blockid BETWEEN b.start_block" + " AND b.leaves_end_block)" + " AND (b.level%%1024)==%d)", + pgsz-45, zTab, zTab, i); + if( sqlite3_step(pStmt)==SQLITE_ROW + && (nLeaf = sqlite3_column_int(pStmt, 0))>0 + ){ + int nIdx = sqlite3_column_int(pStmt, 5); + sqlite3_int64 sz; + printf("For level %d:\n", i); + printf(" Number of indexes...................... %9d\n", nIdx); + printf(" Number of leaf segments................ %9d\n", nLeaf); + if( nIdx>1 ){ + printf(" Average leaf segments per index........ %11.1f\n", + (double)nLeaf/(double)nIdx); + } + printf(" Total size of all leaf segments........ %9lld\n", + (sz = sqlite3_column_int64(pStmt, 1))); + printf(" Average size of leaf segments.......... %11.1f\n", + sqlite3_column_double(pStmt, 2)); + if( nIdx>1 ){ + printf(" Average leaf segment size per index.... %11.1f\n", + (double)sz/(double)nIdx); + } + printf(" Maximum leaf segment size.............. %9lld\n", + sqlite3_column_int64(pStmt, 3)); + n = sqlite3_column_int(pStmt, 4); + printf(" Leaf segments larger than %5d bytes.. %9d %5.2f%%\n", + pgsz-45, n, n*100.0/nLeaf); + } + sqlite3_finalize(pStmt); + } +} + +/* +** Print a single "tree" line of the segdir map output. +*/ +static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){ + printf(" tree %9lld", iLower); + if( iUpper>iLower ){ + printf(" thru %9lld (%lld blocks)", iUpper, iUpper-iLower+1); + } + printf("\n"); +} + +/* +** Check to see if the block of a %_segments entry is NULL. +*/ +static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){ + sqlite3_stmt *pStmt; + int rc = 1; + + pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'" + " WHERE blockid=%lld", zTab, iBlockId); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + rc = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** Show a map of segments derived from the %_segdir table. +*/ +static void showSegdirMap(sqlite3 *db, const char *zTab){ + int mxIndex, iIndex; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pStmt2 = 0; + int prevLevel; + + pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + mxIndex = sqlite3_column_int(pStmt, 0); + }else{ + mxIndex = 0; + } + sqlite3_finalize(pStmt); + + printf("Number of inverted indices............... %3d\n", mxIndex+1); + pStmt = prepare(db, + "SELECT level, idx, start_block, leaves_end_block, end_block, rowid" + " FROM '%q_segdir'" + " WHERE level/1024==?" + " ORDER BY level DESC, idx", + zTab); + pStmt2 = prepare(db, + "SELECT blockid FROM '%q_segments'" + " WHERE blockid BETWEEN ? AND ? ORDER BY blockid", + zTab); + for(iIndex=0; iIndex<=mxIndex; iIndex++){ + if( mxIndex>0 ){ + printf("**************************** Index %d " + "****************************\n", iIndex); + } + sqlite3_bind_int(pStmt, 1, iIndex); + prevLevel = -1; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int iLevel = sqlite3_column_int(pStmt, 0)%1024; + int iIdx = sqlite3_column_int(pStmt, 1); + sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3); + sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4); + char rtag[20]; + if( iLevel!=prevLevel ){ + printf("level %2d idx %2d", iLevel, iIdx); + prevLevel = iLevel; + }else{ + printf(" idx %2d", iIdx); + } + sqlite3_snprintf(sizeof(rtag), rtag, "r%lld", + sqlite3_column_int64(pStmt,5)); + printf(" root %9s\n", rtag); + if( iLEnd>iStart ){ + sqlite3_int64 iLower, iPrev, iX; + if( iLEnd+1<=iEnd ){ + sqlite3_bind_int64(pStmt2, 1, iLEnd+1); + sqlite3_bind_int64(pStmt2, 2, iEnd); + iLower = -1; + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + iX = sqlite3_column_int64(pStmt2, 0); + if( iLower<0 ){ + iLower = iPrev = iX; + }else if( iX==iPrev+1 ){ + iPrev = iX; + }else{ + printTreeLine(iLower, iPrev); + iLower = iPrev = iX; + } + } + sqlite3_reset(pStmt2); + if( iLower>=0 ){ + if( iLower==iPrev && iLower==iEnd + && isNullSegment(db,zTab,iLower) + ){ + printf(" null %9lld\n", iLower); + }else{ + printTreeLine(iLower, iPrev); + } + } + } + printf(" leaves %9lld thru %9lld (%lld blocks)\n", + iStart, iLEnd, iLEnd - iStart + 1); + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + sqlite3_finalize(pStmt2); +} + +/* +** Decode a single segment block and display the results on stdout. +*/ +static void decodeSegment( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iChild; + sqlite3_int64 iPrefix; + sqlite3_int64 nTerm; + sqlite3_int64 n; + sqlite3_int64 iDocsz; + int iHeight; + int i = 0; + int cnt = 0; + char zTerm[1000]; + + i += getVarint(aData, &n); + iHeight = (int)n; + printf("height: %d\n", iHeight); + if( iHeight>0 ){ + i += getVarint(aData+i, &iChild); + printf("left-child: %lld\n", iChild); + } + while( i0 ){ + i += getVarint(aData+i, &iPrefix); + }else{ + iPrefix = 0; + } + i += getVarint(aData+i, &nTerm); + if( iPrefix+nTerm+1 >= sizeof(zTerm) ){ + fprintf(stderr, "term to long\n"); + exit(1); + } + memcpy(zTerm+iPrefix, aData+i, nTerm); + zTerm[iPrefix+nTerm] = 0; + i += nTerm; + if( iHeight==0 ){ + i += getVarint(aData+i, &iDocsz); + printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + i += iDocsz; + }else{ + printf("term: %-25s child %lld\n", zTerm, ++iChild); + } + } +} + + +/* +** Print a a blob as hex and ascii. +*/ +static void printBlob( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + int i, j; + const char *zOfstFmt; + const int perLine = 16; + + if( (nData&~0xfff)==0 ){ + zOfstFmt = " %03x: "; + }else if( (nData&~0xffff)==0 ){ + zOfstFmt = " %04x: "; + }else if( (nData&~0xfffff)==0 ){ + zOfstFmt = " %05x: "; + }else if( (nData&~0xffffff)==0 ){ + zOfstFmt = " %06x: "; + }else{ + zOfstFmt = " %08x: "; + } + + for(i=0; inData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%02x ", aData[i+j]); + } + } + for(j=0; jnData ){ + fprintf(stdout, " "); + }else{ + fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); + } + } + fprintf(stdout,"\n"); + } +} + +/* +** Convert text to a 64-bit integer +*/ +static sqlite3_int64 atoi64(const char *z){ + sqlite3_int64 v = 0; + while( z[0]>='0' && z[0]<='9' ){ + v = v*10 + z[0] - '0'; + z++; + } + return v; +} + +/* +** Return a prepared statement which, when stepped, will return in its +** first column the blob associated with segment zId. If zId begins with +** 'r' then it is a rowid of a %_segdir entry. Otherwise it is a +** %_segment entry. +*/ +static sqlite3_stmt *prepareToGetSegment( + sqlite3 *db, /* The database */ + const char *zTab, /* The FTS3/4 table name */ + const char *zId /* ID of the segment to open */ +){ + sqlite3_stmt *pStmt; + if( zId[0]=='r' ){ + pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld", + zTab, atoi64(zId+1)); + }else{ + pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld", + zTab, atoi64(zId)); + } + return pStmt; +} + +/* +** Print the content of a segment or of the root of a segdir. The segment +** or root is identified by azExtra[0]. If the first character of azExtra[0] +** is 'r' then the remainder is the integer rowid of the %_segdir entry. +** If the first character of azExtra[0] is not 'r' then, then all of +** azExtra[0] is an integer which is the block number. +** +** If the --raw option is present in azExtra, then a hex dump is provided. +** Otherwise a decoding is shown. +*/ +static void showSegment(sqlite3 *db, const char *zTab){ + const unsigned char *aData; + int nData; + sqlite3_stmt *pStmt; + + pStmt = prepareToGetSegment(db, zTab, azExtra[0]); + if( sqlite3_step(pStmt)!=SQLITE_ROW ){ + sqlite3_finalize(pStmt); + return; + } + nData = sqlite3_column_bytes(pStmt, 0); + aData = sqlite3_column_blob(pStmt, 0); + printf("Segment %s of size %d bytes:\n", azExtra[0], nData); + if( findOption("raw", 0, 0)!=0 ){ + printBlob(aData, nData); + }else{ + decodeSegment(aData, nData); + } + sqlite3_finalize(pStmt); +} + +/* +** Decode a single doclist and display the results on stdout. +*/ +static void decodeDoclist( + const unsigned char *aData, /* Content to print */ + int nData /* Number of bytes of content */ +){ + sqlite3_int64 iPrevDocid = 0; + sqlite3_int64 iDocid; + sqlite3_int64 iPos; + sqlite3_int64 iPrevPos = 0; + sqlite3_int64 iCol; + int i = 0; + + while( ipKeyInfo==0) ); - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, nKey, 0); - } - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -6809,6 +6802,14 @@ int sqlite3BtreeInsert( */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced (assuming this is a replace + ** operation - if it is not, the following is a no-op). */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, nKey, 0); + } + if( !loc ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; @@ -6919,12 +6920,6 @@ int sqlite3BtreeDelete(BtCursor *pCur){ return SQLITE_ERROR; /* Something has gone awry. */ } - /* If this is a delete operation to remove a row from a table b-tree, - ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; pPage = pCur->apPage[iCellDepth]; @@ -6950,6 +6945,13 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 ){ + invalidateIncrblobCursors(p, pCur->info.nKey, 0); + } + rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell); @@ -7231,13 +7233,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); - /* Invalidate all incrblob cursors open on table iTable (assuming iTable - ** is the root of a table b-tree - if it is not, the following call is - ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); - rc = saveAllCursors(pBt, (Pgno)iTable, 0); + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + invalidateIncrblobCursors(p, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); diff --git a/src/build.c b/src/build.c index daa5430406..6fc59dade4 100644 --- a/src/build.c +++ b/src/build.c @@ -537,7 +537,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(db, pTable->pCheck); + sqlite3ExprListDelete(db, pTable->pCheck); #endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); @@ -1200,15 +1200,17 @@ void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ - sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; if( pTab && !IN_DECLARE_VTAB ){ - pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, pCheckExpr); + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + } }else #endif { - sqlite3ExprDelete(db, pCheckExpr); + sqlite3ExprDelete(pParse->db, pCheckExpr); } } @@ -1478,6 +1480,8 @@ void sqlite3EndTable( if( p->pCheck ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + ExprList *pList; /* List of all CHECK constraints */ + int i; /* Loop counter */ memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); @@ -1488,8 +1492,11 @@ void sqlite3EndTable( sNC.pParse = pParse; sNC.pSrcList = &sSrc; sNC.isCheck = 1; - if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ - return; + pList = p->pCheck; + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ + return; + } } } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -3040,19 +3047,21 @@ exit_drop_index: } /* -** pArray is a pointer to an array of objects. Each object in the -** array is szEntry bytes in size. This routine allocates a new -** object on the end of the array. +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. ** -** *pnEntry is the number of entries already in use. *pnAlloc is -** the previously allocated size of the array. initSize is the -** suggested initial array size allocation. +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). ** -** The index of the new entry is returned in *pIdx. +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. ** -** This routine returns a pointer to the array of objects. This -** might be the same as the pArray parameter or it might be a different -** pointer if the array was resized. +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. */ void *sqlite3ArrayAllocate( sqlite3 *db, /* Connection to notify of malloc failures */ diff --git a/src/delete.c b/src/delete.c index c8a3c6e7fd..b04e5a1508 100644 --- a/src/delete.c +++ b/src/delete.c @@ -381,7 +381,7 @@ void sqlite3DeleteFrom( pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK ); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); diff --git a/src/expr.c b/src/expr.c index 79dd8f4961..5e3f1204a1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2032,15 +2032,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -2175,7 +2166,8 @@ int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -2190,7 +2182,11 @@ int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -2318,7 +2314,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; }else{ inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + pExpr->iColumn, pExpr->iTable, target, + pExpr->op2); } break; } @@ -2595,6 +2592,25 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->flags==SQLITE_FUNC_LENGTH ); + pFarg->a[0].pExpr->op2 = pDef->flags; + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ diff --git a/src/func.c b/src/func.c index c66ad28aba..6ffc7184b0 100644 --- a/src/func.c +++ b/src/func.c @@ -1542,8 +1542,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), @@ -1555,11 +1555,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/insert.c b/src/insert.c index 463b948c82..3d521ab29a 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1157,9 +1157,11 @@ void sqlite3GenerateConstraintChecks( int regData; /* Register containing first data column */ int iCur; /* Table cursor number */ Index *pIdx; /* Pointer to one of the indices */ + sqlite3 *db; /* Database connection */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int regOldRowid = (rowidChng && isUpdate) ? rowidChng : regRowid; + db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1192,7 +1194,7 @@ void sqlite3GenerateConstraintChecks( char *zMsg; sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT, onError, regData+i); - zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + zMsg = sqlite3MPrintf(db, "%s.%s may not be NULL", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); break; @@ -1214,18 +1216,28 @@ void sqlite3GenerateConstraintChecks( /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK - if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ - int allOk = sqlite3VdbeMakeLabel(v); + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + int i; pParse->ckBase = regData; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; - if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); - }else{ - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ - sqlite3HaltConstraint(pParse, onError, 0, 0); + for(i=0; inExpr; i++){ + int allOk = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); + if( onError==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + }else{ + char *zConsName = pCheck->a[i].zName; + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( zConsName ){ + zConsName = sqlite3MPrintf(db, "constraint %s failed", zConsName); + }else{ + zConsName = 0; + } + sqlite3HaltConstraint(pParse, onError, zConsName, P4_DYNAMIC); + } + sqlite3VdbeResolveLabel(v, allOk); } - sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -1281,7 +1293,7 @@ void sqlite3GenerateConstraintChecks( ** table. */ Trigger *pTrigger = 0; - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ @@ -1380,7 +1392,7 @@ void sqlite3GenerateConstraintChecks( char *zErr; sqlite3StrAccumInit(&errMsg, 0, 0, 200); - errMsg.db = pParse->db; + errMsg.db = db; zSep = pIdx->nColumn>1 ? "columns " : "column "; for(j=0; jnColumn; j++){ char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; @@ -1404,7 +1416,7 @@ void sqlite3GenerateConstraintChecks( Trigger *pTrigger = 0; assert( onError==OE_Replace ); sqlite3MultiWrite(pParse); - if( pParse->db->flags&SQLITE_RecTriggers ){ + if( db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete( @@ -1734,7 +1746,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck, pDest->pCheck) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/src/os_win.c b/src/os_win.c index 8b86c7ed59..358b70c128 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1616,6 +1616,9 @@ static int winClose(sqlite3_file *id){ } #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); + if( rc ){ + pFile->h = NULL; + } OpenCounter(-1); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), @@ -1633,6 +1636,9 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ int nRetry = 0; /* Number of retrys */ @@ -1641,10 +1647,18 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; @@ -1671,7 +1685,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc; /* True if error has occured, else false */ + int rc = 0; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -1682,19 +1696,44 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ +#if !SQLITE_OS_WINCE + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + while( nRem>0 ){ +#if SQLITE_OS_WINCE if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif if( retryIoerr(&nRetry, &lastErrno) ) continue; break; } - if( nWrite<=0 ) break; + if( nWrite<=0 ){ + lastErrno = osGetLastError(); + break; + } +#if !SQLITE_OS_WINCE + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif aRem += nWrite; nRem -= nWrite; } diff --git a/src/pager.c b/src/pager.c index 66252aa0f7..2d603122d5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -670,9 +670,9 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int nHit, nMiss; /* Total cache hits and misses */ + int aStat[3]; /* Total cache hits, misses and writes */ #ifdef SQLITE_TEST - int nRead, nWrite; /* Database pages read/written */ + int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC @@ -689,6 +689,15 @@ struct Pager { #endif }; +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 + /* ** The following global variables hold counters used for ** testing purposes only. These variables do not exist in @@ -2971,6 +2980,7 @@ static int pagerWalFrames( int isCommit /* True if this is a commit */ ){ int rc; /* Return code */ + int nList; /* Number of pages in pList */ #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES) PgHdr *p; /* For looping over pages */ #endif @@ -2984,6 +2994,7 @@ static int pagerWalFrames( } #endif + assert( pList->pDirty==0 || isCommit ); if( isCommit ){ /* If a WAL transaction is being committed, there is no point in writing ** any pages with page numbers greater than nTruncate into the WAL file. @@ -2991,11 +3002,18 @@ static int pagerWalFrames( ** list here. */ PgHdr *p; PgHdr **ppNext = &pList; + nList = 0; for(p=pList; (*ppNext = p); p=p->pDirty){ - if( p->pgno<=nTruncate ) ppNext = &p->pDirty; + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } } assert( pList ); + }else{ + nList = 1; } + pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); rc = sqlite3WalFrames(pPager->pWal, @@ -4063,6 +4081,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } + pPager->aStat[PAGER_STAT_WRITE]++; /* Update any backup objects copying the contents of this pager. */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); @@ -4071,7 +4090,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ PAGERID(pPager), pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pgno)); PAGER_INCR(sqlite3_pager_writedb_count); - PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } @@ -5029,7 +5047,7 @@ int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - pPager->nHit++; + pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ @@ -5071,7 +5089,7 @@ int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); - pPager->nMiss++; + pPager->aStat[PAGER_STAT_MISS]++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -5656,6 +5674,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ pPager->changeCountDone = 1; @@ -6099,11 +6118,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->nHit; - a[7] = pPager->nMiss; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->nWrite; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; return a; } #endif @@ -6116,20 +6135,19 @@ int *sqlite3PagerStats(Pager *pPager){ ** returning. */ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ - int *piStat; assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE ); - if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ - piStat = &pPager->nHit; - }else{ - piStat = &pPager->nMiss; - } - *pnVal += *piStat; + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + + *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; if( reset ){ - *piStat = 0; + pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; } } diff --git a/src/parse.y b/src/parse.y index ed18e7f973..49406e1e76 100644 --- a/src/parse.y +++ b/src/parse.y @@ -75,7 +75,7 @@ struct LimitVal { */ struct LikeOp { Token eOperator; /* "like" or "glob" or "regexp" */ - int not; /* True if the NOT keyword is present */ + int bNot; /* True if the NOT keyword is present */ }; /* @@ -273,10 +273,10 @@ signed ::= minus_num. // "carglist" is a list of additional constraints that come after the // column name and column type in a CREATE TABLE statement. // -carglist ::= carglist carg. +carglist ::= carglist cname ccons. carglist ::= . -carg ::= CONSTRAINT nm ccons. -carg ::= ccons. +cname ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +cname ::= . {pParse->constraintName.n = 0;} ccons ::= DEFAULT term(X). {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT LP expr(X) RP. {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT PLUS term(X). {sqlite3AddDefaultValue(pParse,&X);} @@ -339,15 +339,10 @@ init_deferred_pred_opt(A) ::= . {A = 0;} init_deferred_pred_opt(A) ::= INITIALLY DEFERRED. {A = 1;} init_deferred_pred_opt(A) ::= INITIALLY IMMEDIATE. {A = 0;} -// For the time being, the only constraint we care about is the primary -// key and UNIQUE. Both create indices. -// conslist_opt(A) ::= . {A.n = 0; A.z = 0;} conslist_opt(A) ::= COMMA(X) conslist. {A = X;} -conslist ::= conslist COMMA tcons. -conslist ::= conslist tcons. -conslist ::= tcons. -tcons ::= CONSTRAINT nm. +conslist ::= conslist COMMA cname tcons. +conslist ::= cname tcons. tcons ::= PRIMARY KEY LP idxlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP idxlist(X) RP onconf(R). @@ -881,16 +876,16 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.not = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.not = 1;} +likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; @@ -901,7 +896,7 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); + if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0); A.zStart = X.zStart; A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; diff --git a/src/select.c b/src/select.c index 3efe014d8b..c225013672 100644 --- a/src/select.c +++ b/src/select.c @@ -4228,7 +4228,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } diff --git a/src/shell.c b/src/shell.c index d3ddfa9536..4287ef17d8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -68,6 +68,8 @@ # include #define isatty(h) _isatty(h) #define access(f,m) _access((f),(m)) +#define popen(a,b) _popen((a),(b)) +#define pclose(x) _pclose(x) #else /* Make sure isatty() has a prototype. */ @@ -1077,6 +1079,9 @@ static int display_stats( sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + fprintf(pArg->out, "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; @@ -1287,7 +1292,6 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ char *zTableInfo = 0; char *zTmp = 0; int nRow = 0; - int kk; zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); zTableInfo = appendText(zTableInfo, zTable, '"'); @@ -1300,12 +1304,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ } zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); - if( !isalpha(zTable[0]) ){ - kk = 0; - }else{ - for(kk=1; isalnum(zTable[kk]); kk++){} - } - zTmp = appendText(zTmp, zTable, zTable[kk] ? '"' : 0); + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + zTmp = appendText(zTmp, zTable, '"'); if( zTmp ){ zSelect = appendText(zSelect, zTmp, '\''); } @@ -2000,11 +2001,24 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ if( p->out!=stdout ){ - fclose(p->out); + if( p->outfile[0]=='|' ){ + pclose(p->out); + }else{ + fclose(p->out); + } } if( strcmp(azArg[1],"stdout")==0 ){ p->out = stdout; sqlite3_snprintf(sizeof(p->outfile), p->outfile, "stdout"); + }else if( azArg[1][0]=='|' ){ + p->out = popen(&azArg[1][1], "w"); + if( p->out==0 ){ + fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]); + p->out = stdout; + rc = 1; + }else{ + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); + } }else{ p->out = fopen(azArg[1], "wb"); if( p->out==0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 757e8514a5..8cd067651d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6001,6 +6001,17 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. ** +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
SQLITE_DBSTATUS_CACHE_WRITE
+**
This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined). ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6012,7 +6023,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 #define SQLITE_DBSTATUS_CACHE_HIT 7 #define SQLITE_DBSTATUS_CACHE_MISS 8 -#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_MAX 9 /* Largest defined DBSTATUS */ /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 33d4f65db1..526de1eb4a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1017,14 +1017,18 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ #define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ #define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_COUNT 0x10 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_LENGTH 0x40 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x80 /* Built-in typeof() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1052,7 +1056,10 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ @@ -1282,7 +1289,7 @@ struct Table { FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ #ifndef SQLITE_OMIT_CHECK - Expr *pCheck; /* The AND of all CHECK constraints */ + ExprList *pCheck; /* All CHECK constraints */ #endif #ifndef SQLITE_OMIT_ALTERTABLE int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ @@ -1675,6 +1682,7 @@ struct Expr { i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ + /* If TK_COLUMN, the value of p5 for OP_Column */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ #if SQLITE_MAX_EXPR_DEPTH>0 @@ -1697,7 +1705,7 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Hint 0x1000 /* Optimizer hint. Not required for correctness */ +#define EP_Hint 0x1000 /* Not used */ #define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ #define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ @@ -2216,6 +2224,7 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ + Token constraintName;/* Name of the constraint currently being parsed */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -2284,7 +2293,10 @@ struct AuthContext { }; /* -** Bitfield flags for P5 value in OP_Insert and OP_Delete +** Bitfield flags for P5 value in various opcodes. +** +** Note that the values for ISNOOP and LENGTHARG are the same. But as +** those bits are never used on the same opcode, the overlap is harmless. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ @@ -2293,6 +2305,8 @@ struct AuthContext { #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ /* * Each trigger present in the database schema is stored as an instance of @@ -2776,7 +2790,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); diff --git a/src/status.c b/src/status.c index b23238bb14..04b7656afe 100644 --- a/src/status.c +++ b/src/status.c @@ -224,10 +224,12 @@ int sqlite3_db_status( ** to zero. */ case SQLITE_DBSTATUS_CACHE_HIT: - case SQLITE_DBSTATUS_CACHE_MISS: { + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; int nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); for(i=0; inDb; i++){ if( db->aDb[i].pBt ){ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index f2a517c8f6..cc928ff6e8 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3254,23 +3254,19 @@ EXTERN int Sqlite3_Init(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; } EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} +/* Because it accesses the file-system and uses persistent state, SQLite +** is not considered appropriate for safe interpreters. Hence, we deliberately +** omit the _SafeInit() interfaces. +*/ #ifndef SQLITE_3_SUFFIX_ONLY int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } -int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; } int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;} #endif #ifdef TCLSH diff --git a/src/test1.c b/src/test1.c index e1220cf11f..eed6474b90 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3263,7 +3263,7 @@ static int test_bind_text16( char *value; int rc; - void (*xDel)() = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); + void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); Tcl_Obj *oStmt = objv[objc-4]; Tcl_Obj *oN = objv[objc-3]; Tcl_Obj *oString = objv[objc-2]; @@ -3611,7 +3611,7 @@ static int test_prepare( if( bytes>=0 ){ bytes = bytes - (zTail-zSql); } - if( strlen(zTail) /* ** The background thread that does file locking. */ @@ -6128,7 +6129,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA {"sqlite3_column_database_name16", - test_stmt_utf16, sqlite3_column_database_name16}, + test_stmt_utf16, (void*)sqlite3_column_database_name16}, {"sqlite3_column_table_name16", test_stmt_utf16, (void*)sqlite3_column_table_name16}, {"sqlite3_column_origin_name16", test_stmt_utf16, (void*)sqlite3_column_origin_name16}, #endif diff --git a/src/test3.c b/src/test3.c index 9bac2cac13..947ded7d91 100644 --- a/src/test3.c +++ b/src/test3.c @@ -465,7 +465,7 @@ static int btree_varint_test( if( Tcl_GetInt(interp, argv[4], (int*)&incr) ) return TCL_ERROR; in = start; in *= mult; - for(i=0; i9 || n1<1 ){ diff --git a/src/test6.c b/src/test6.c index 5f64cacca0..e2cee09193 100644 --- a/src/test6.c +++ b/src/test6.c @@ -177,7 +177,7 @@ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ iSkip = 512; } if( (iAmt-iSkip)>0 ){ - rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], iAmt-iSkip, iOff+iSkip); + rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], (int)(iAmt-iSkip), iOff+iSkip); } return rc; } @@ -306,8 +306,8 @@ static int writeListSync(CrashFile *pFile, int isCrash){ } case 3: { /* Trash sectors */ u8 *zGarbage; - int iFirst = (pWrite->iOffset/g.iSectorSize); - int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize; + int iFirst = (int)(pWrite->iOffset/g.iSectorSize); + int iLast = (int)((pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize); assert(pWrite->zBuf); @@ -430,7 +430,7 @@ static int cfWrite( ){ CrashFile *pCrash = (CrashFile *)pFile; if( iAmt+iOfst>pCrash->iSize ){ - pCrash->iSize = iAmt+iOfst; + pCrash->iSize = (int)(iAmt+iOfst); } while( pCrash->iSize>pCrash->nData ){ u8 *zNew; @@ -454,7 +454,7 @@ static int cfTruncate(sqlite3_file *pFile, sqlite_int64 size){ CrashFile *pCrash = (CrashFile *)pFile; assert(size>=0); if( pCrash->iSize>size ){ - pCrash->iSize = size; + pCrash->iSize = (int)size; } return writeListAppend(pFile, size, 0, 0); } @@ -518,7 +518,7 @@ static int cfFileControl(sqlite3_file *pFile, int op, void *pArg){ i64 nByte = *(i64 *)pArg; if( nByte>pCrash->iSize ){ if( SQLITE_OK==writeListAppend(pFile, nByte, 0, 0) ){ - pCrash->iSize = nByte; + pCrash->iSize = (int)nByte; } } return SQLITE_OK; @@ -635,11 +635,11 @@ static int cfOpen( iChunk = PENDING_BYTE; } memset(pWrapper->zData, 0, pWrapper->nData); - rc = sqlite3OsRead(pReal, pWrapper->zData, iChunk, 0); + rc = sqlite3OsRead(pReal, pWrapper->zData, (int)iChunk, 0); if( SQLITE_OK==rc && pWrapper->iSize>(PENDING_BYTE+512) && isDb ){ i64 iOff = PENDING_BYTE+512; iChunk = pWrapper->iSize - iOff; - rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], iChunk, iOff); + rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], (int)iChunk, iOff); } }else{ rc = SQLITE_NOMEM; diff --git a/src/test8.c b/src/test8.c index 283d790b7d..80e7adaf2c 100644 --- a/src/test8.c +++ b/src/test8.c @@ -831,13 +831,10 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ if( !isIgnoreUsable && !pConstraint->usable ) continue; iCol = pConstraint->iColumn; - if( pVtab->aIndex[iCol] || iCol<0 ){ - char *zCol = pVtab->aCol[iCol]; + if( iCol<0 || pVtab->aIndex[iCol] ){ + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zOp = 0; useIdx = 1; - if( iCol<0 ){ - zCol = "rowid"; - } switch( pConstraint->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: zOp = "="; break; @@ -870,13 +867,12 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** on a column that this virtual table has an index for, then consume ** the ORDER BY clause. */ - if( pIdxInfo->nOrderBy==1 && pVtab->aIndex[pIdxInfo->aOrderBy->iColumn] ){ + if( pIdxInfo->nOrderBy==1 && ( + pIdxInfo->aOrderBy->iColumn<0 || + pVtab->aIndex[pIdxInfo->aOrderBy->iColumn]) ){ int iCol = pIdxInfo->aOrderBy->iColumn; - char *zCol = pVtab->aCol[iCol]; + char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid"; char *zDir = pIdxInfo->aOrderBy->desc?"DESC":"ASC"; - if( iCol<0 ){ - zCol = "rowid"; - } zNew = sqlite3_mprintf(" ORDER BY %s %s", zCol, zDir); string_concat(&zQuery, zNew, 1, &rc); pIdxInfo->orderByConsumed = 1; diff --git a/src/test_journal.c b/src/test_journal.c index 00567db598..4cb51fbcf5 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -404,7 +404,7 @@ static int openTransaction(jt_file *pMain, jt_file *pJournal){ /* Calculate and store a checksum for each page in the database file. */ if( rc==SQLITE_OK ){ int ii; - for(ii=0; rc==SQLITE_OK && iinPage; ii++){ + for(ii=0; rc==SQLITE_OK && ii<(int)pMain->nPage; ii++){ i64 iOff = (i64)(pMain->nPagesize) * (i64)ii; if( iOff==PENDING_BYTE ) continue; rc = sqlite3OsRead(pMain->pReal, aData, pMain->nPagesize, iOff); @@ -466,7 +466,7 @@ static int readJournalFile(jt_file *p, jt_file *pMain){ continue; } } - nRec = (iSize-iOff) / (pMain->nPagesize+8); + nRec = (u32)((iSize-iOff) / (pMain->nPagesize+8)); } /* Read all the records that follow the journal-header just read. */ @@ -538,7 +538,7 @@ static int jtWrite( } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ - if( iAmtnPagesize + if( iAmt<(int)p->nPagesize && p->nPagesize%iAmt==0 && iOfst>=(PENDING_BYTE+512) && iOfst+iAmt<=PENDING_BYTE+p->nPagesize @@ -549,7 +549,7 @@ static int jtWrite( ** pending-byte page. */ }else{ - u32 pgno = iOfst/p->nPagesize + 1; + u32 pgno = (u32)(iOfst/p->nPagesize + 1); assert( (iAmt==1||iAmt==p->nPagesize) && ((iOfst+iAmt)%p->nPagesize)==0 ); assert( pgno<=p->nPage || p->nSync>0 ); assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) ); @@ -578,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ u32 pgno; u32 locking_page = (u32)(PENDING_BYTE/p->nPagesize+1); - for(pgno=size/p->nPagesize+1; pgno<=p->nPage; pgno++){ + for(pgno=(u32)(size/p->nPagesize+1); pgno<=p->nPage; pgno++){ assert( pgno==locking_page || sqlite3BitvecTest(p->pWritable, pgno) ); } } diff --git a/src/test_malloc.c b/src/test_malloc.c index 037dfd140a..09b8f738e8 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1323,7 +1323,8 @@ static int test_db_status( { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE }, { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL }, { "CACHE_HIT", SQLITE_DBSTATUS_CACHE_HIT }, - { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS } + { "CACHE_MISS", SQLITE_DBSTATUS_CACHE_MISS }, + { "CACHE_WRITE", SQLITE_DBSTATUS_CACHE_WRITE } }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/src/test_multiplex.c b/src/test_multiplex.c index e2b6720652..32dd8e33e8 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -529,7 +529,7 @@ static int multiplexOpen( pGroup->bEnabled = -1; pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", (flags & SQLITE_OPEN_MAIN_DB)==0); - pGroup->szChunk = sqlite3_uri_int64(zUri, "chunksize", + pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize", SQLITE_MULTIPLEX_CHUNK_SIZE); pGroup->szChunk = (pGroup->szChunk+0xffff)&~0xffff; if( zName ){ @@ -597,7 +597,7 @@ static int multiplexOpen( bExists = multiplexSubSize(pGroup, 1, &rc)>0; if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ - pGroup->szChunk = sz; + pGroup->szChunk = (int)sz; }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ pGroup->bEnabled = 0; } diff --git a/src/test_onefile.c b/src/test_onefile.c index cd7db008cf..adc0d13d9f 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -288,7 +288,7 @@ static int tmpWrite( ){ tmp_file *pTmp = (tmp_file *)pFile; if( (iAmt+iOfst)>pTmp->nAlloc ){ - int nNew = 2*(iAmt+iOfst+pTmp->nAlloc); + int nNew = (int)(2*(iAmt+iOfst+pTmp->nAlloc)); char *zNew = sqlite3_realloc(pTmp->zAlloc, nNew); if( !zNew ){ return SQLITE_NOMEM; @@ -297,7 +297,7 @@ static int tmpWrite( pTmp->nAlloc = nNew; } memcpy(&pTmp->zAlloc[iOfst], zBuf, iAmt); - pTmp->nSize = MAX(pTmp->nSize, iOfst+iAmt); + pTmp->nSize = (int)MAX(pTmp->nSize, iOfst+iAmt); return SQLITE_OK; } @@ -306,7 +306,7 @@ static int tmpWrite( */ static int tmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ tmp_file *pTmp = (tmp_file *)pFile; - pTmp->nSize = MIN(pTmp->nSize, size); + pTmp->nSize = (int)MIN(pTmp->nSize, size); return SQLITE_OK; } @@ -418,7 +418,7 @@ static int fsRead( /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -453,14 +453,14 @@ static int fsWrite( }else{ rc = pF->pMethods->xWrite(pF, zBuf, iAmt, iOfst+BLOCKSIZE); if( rc==SQLITE_OK ){ - pReal->nDatabase = MAX(pReal->nDatabase, iAmt+iOfst); + pReal->nDatabase = (int)MAX(pReal->nDatabase, iAmt+iOfst); } } }else{ /* Journal file. */ int iRem = iAmt; int iBuf = 0; - int ii = iOfst; + int ii = (int)iOfst; while( iRem>0 && rc==SQLITE_OK ){ int iRealOff = pReal->nBlob - BLOCKSIZE*((ii/BLOCKSIZE)+1) + ii%BLOCKSIZE; int iRealAmt = MIN(iRem, BLOCKSIZE - (iRealOff%BLOCKSIZE)); @@ -475,7 +475,7 @@ static int fsWrite( } } if( rc==SQLITE_OK ){ - pReal->nJournal = MAX(pReal->nJournal, iAmt+iOfst); + pReal->nJournal = (int)MAX(pReal->nJournal, iAmt+iOfst); } } @@ -489,9 +489,9 @@ static int fsTruncate(sqlite3_file *pFile, sqlite_int64 size){ fs_file *p = (fs_file *)pFile; fs_real_file *pReal = p->pReal; if( p->eType==DATABASE_FILE ){ - pReal->nDatabase = MIN(pReal->nDatabase, size); + pReal->nDatabase = (int)MIN(pReal->nDatabase, size); }else{ - pReal->nJournal = MIN(pReal->nJournal, size); + pReal->nJournal = (int)MIN(pReal->nJournal, size); } return SQLITE_OK; } @@ -641,7 +641,7 @@ static int fsOpen( pReal->nBlob = BLOBSIZE; }else{ unsigned char zS[4]; - pReal->nBlob = size; + pReal->nBlob = (int)size; rc = pRealFile->pMethods->xRead(pRealFile, zS, 4, 0); pReal->nDatabase = (zS[0]<<24)+(zS[1]<<16)+(zS[2]<<8)+zS[3]; if( rc==SQLITE_OK ){ diff --git a/src/test_osinst.c b/src/test_osinst.c index b7f350577c..35e168c418 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -242,7 +242,7 @@ static sqlite3_uint64 vfslog_time(){ } #endif -static void vfslog_call(sqlite3_vfs *, int, int, int, int, int, int); +static void vfslog_call(sqlite3_vfs *, int, int, sqlite3_int64, int, int, int); static void vfslog_string(sqlite3_vfs *, const char *); /* @@ -648,7 +648,7 @@ static void vfslog_call( sqlite3_vfs *pVfs, int eEvent, int iFileid, - int nClick, + sqlite3_int64 nClick, int return_code, int size, int offset @@ -661,7 +661,7 @@ static void vfslog_call( zRec = (unsigned char *)&p->aBuf[p->nBuf]; put32bits(&zRec[0], eEvent); put32bits(&zRec[4], iFileid); - put32bits(&zRec[8], nClick); + put32bits(&zRec[8], (unsigned int)(nClick&0xffff)); put32bits(&zRec[12], return_code); put32bits(&zRec[16], size); put32bits(&zRec[20], offset); @@ -1043,7 +1043,7 @@ static int vlogColumn( } case 1: { char *zStr = pCsr->zTransient; - if( val!=0 && valnFile ){ + if( val!=0 && val<(unsigned)pCsr->nFile ){ zStr = pCsr->azFile[val]; } sqlite3_result_text(ctx, zStr, -1, SQLITE_TRANSIENT); diff --git a/src/test_quota.c b/src/test_quota.c index c8ed7389e5..e749851238 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -1060,7 +1060,7 @@ size_t sqlite3_quota_fwrite( } if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){ iEnd = pGroup->iLimit - pGroup->iSize + pFile->iSize; - nmemb = (iEnd - iOfst)/size; + nmemb = (size_t)((iEnd - iOfst)/size); iEnd = iOfst + size*nmemb; szNew = pGroup->iSize - pFile->iSize + iEnd; } diff --git a/src/test_stat.c b/src/test_stat.c index a0893b3974..e83cebebe0 100644 --- a/src/test_stat.c +++ b/src/test_stat.c @@ -324,12 +324,13 @@ static int statDecodePage(Btree *pBt, StatPage *p){ u64 dummy; iOff += sqlite3GetVarint(&aData[iOff], &dummy); } - if( nPayload>p->nMxPayload ) p->nMxPayload = nPayload; + if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; getLocalPayload(nUsable, p->flags, nPayload, &nLocal); pCell->nLocal = nLocal; + assert( nLocal>=0 ); assert( nPayload>=nLocal ); assert( nLocal<=(nUsable-35) ); - if( nPayload>nLocal ){ + if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); @@ -378,7 +379,7 @@ static void statSizeAndOffset(StatCursor *pCsr){ x[0] = pCsr->iPageno; if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ pCsr->iOffset = x[0]; - pCsr->szPage = x[1]; + pCsr->szPage = (int)x[1]; } } @@ -400,7 +401,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ int nPage; - u32 iRoot = sqlite3_column_int64(pCsr->pStmt, 1); + u32 iRoot = (u32)sqlite3_column_int64(pCsr->pStmt, 1); sqlite3PagerPagecount(pPager, &nPage); if( nPage==0 ){ pCsr->isEof = 1; diff --git a/src/test_wholenumber.c b/src/test_wholenumber.c index 150dc95ac4..7c42d01691 100644 --- a/src/test_wholenumber.c +++ b/src/test_wholenumber.c @@ -33,8 +33,8 @@ typedef struct wholenumber_cursor wholenumber_cursor; struct wholenumber_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - unsigned iValue; /* Current value */ - unsigned mxValue; /* Maximum value */ + sqlite3_int64 iValue; /* Current value */ + sqlite3_int64 mxValue; /* Maximum value */ }; /* Methods for the wholenumber module */ diff --git a/src/util.c b/src/util.c index 325c75aae1..dd3b08ae46 100644 --- a/src/util.c +++ b/src/util.c @@ -216,11 +216,11 @@ int sqlite3Dequote(char *z){ ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. ** -** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows -** applications and extensions to compare the contents of two buffers -** containing UTF-8 strings in a case-independent fashion, using the same -** definition of case independence that SQLite uses internally when -** comparing identifiers. +** IMPLEMENTATION-OF: R-30243-02494 The sqlite3_stricmp() and +** sqlite3_strnicmp() APIs allow applications and extensions to compare +** the contents of two buffers containing UTF-8 strings in a +** case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; diff --git a/src/vdbe.c b/src/vdbe.c index ca70ea7a79..8b90f25360 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2137,6 +2137,11 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** then the cache of the cursor is reset prior to extracting the column. ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2277,7 +2282,7 @@ case OP_Column: { pC->aRow = 0; } } - /* The following assert is true in all cases accept when + /* The following assert is true in all cases except when ** the database file has been corrupted externally. ** assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */ szHdr = getVarint32((u8*)zData, offset); @@ -2352,11 +2357,11 @@ case OP_Column: { break; } }else{ - /* If i is less that nField, then there are less fields in this + /* If i is less that nField, then there are fewer fields in this ** record than SetNumColumns indicated there are columns in the ** table. Set the offset for any extra columns not present in - ** the record to 0. This tells code below to store a NULL - ** instead of deserializing a value from the record. + ** the record to 0. This tells code below to store the default value + ** for the column instead of deserializing a value from the record. */ aOffset[i] = 0; } @@ -2386,17 +2391,32 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ + /* This is the common case where the whole row fits on a single page */ VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ - len = sqlite3VdbeSerialTypeLen(aType[p2]); - sqlite3VdbeMemMove(&sMem, pDest); - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; + /* This branch happens only when the row overflows onto multiple pages */ + t = aType[p2]; + if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(X) function if X is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. */ + zData = t<12 ? (char*)&payloadSize64 : 0; + }else{ + len = sqlite3VdbeSerialTypeLen(t); + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + zData = sMem.z; } - zData = sMem.z; - sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); + sqlite3VdbeSerialGet((u8*)zData, t, pDest); } pDest->enc = encoding; }else{ diff --git a/src/where.c b/src/where.c index e6083f0033..5471f71dcf 100644 --- a/src/where.c +++ b/src/where.c @@ -686,7 +686,10 @@ static int isLikeOrGlob( #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; @@ -4383,7 +4386,7 @@ static Bitmask codeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); + regRowid, 0); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } diff --git a/test/backcompat.test b/test/backcompat.test index e8e2f61581..6a0f91ae97 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -27,37 +27,13 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl +source $testdir/bc_common.tcl db close -# Search for binaries to test against. Any executable files that match -# our naming convention are assumed to be testfixture binaries to test -# against. -# -set binaries [list] -set pattern "[file tail [info nameofexec]]?*" -if {$tcl_platform(platform)=="windows"} { - set pattern [string map {\.exe {}} $pattern] -} -foreach file [glob -nocomplain $pattern] { - if {[file executable $file] && [file isfile $file]} {lappend binaries $file} -} -if {[llength $binaries]==0} { - puts "WARNING: No historical binaries to test against." - puts "WARNING: No backwards-compatibility tests have been run." +if {"" == [bc_find_binaries backcompat.test]} { finish_test return } -proc get_version {binary} { - set chan [launch_testfixture $binary] - set v [testfixture $chan { sqlite3 -version }] - close $chan - set v -} -foreach bin $binaries { - puts -nonewline "Testing against $bin - " - flush stdout - puts "version [get_version $bin]" -} proc do_backcompat_test {rv bin1 bin2 script} { @@ -93,7 +69,7 @@ proc do_backcompat_test {rv bin1 bin2 script} { array set ::incompatible [list] proc do_allbackcompat_test {script} { - foreach bin $::binaries { + foreach bin $::BC(binaries) { set nErr [set_test_counter errors] foreach dir {0 1} { diff --git a/test/bc_common.tcl b/test/bc_common.tcl new file mode 100644 index 0000000000..eb9b6db9d3 --- /dev/null +++ b/test/bc_common.tcl @@ -0,0 +1,72 @@ + + + +proc bc_find_binaries {zCaption} { + # Search for binaries to test against. Any executable files that match + # our naming convention are assumed to be testfixture binaries to test + # against. + # + set binaries [list] + set pattern "[file tail [info nameofexec]]?*" + if {$::tcl_platform(platform)=="windows"} { + set pattern [string map {\.exe {}} $pattern] + } + foreach file [glob -nocomplain $pattern] { + if {[file executable $file] && [file isfile $file]} {lappend binaries $file} + } + + if {[llength $binaries]==0} { + puts "WARNING: No historical binaries to test against." + puts "WARNING: Omitting backwards-compatibility tests" + } + + foreach bin $binaries { + puts -nonewline "Testing against $bin - " + flush stdout + puts "version [get_version $bin]" + } + + set ::BC(binaries) $binaries + return $binaries +} + +proc get_version {binary} { + set chan [launch_testfixture $binary] + set v [testfixture $chan { sqlite3 -version }] + close $chan + set v +} + +proc do_bc_test {bin script} { + + forcedelete test.db + set ::bc_chan [launch_testfixture $bin] + + proc code1 {tcl} { uplevel #0 $tcl } + proc code2 {tcl} { testfixture $::bc_chan $tcl } + proc sql1 sql { code1 [list db eval $sql] } + proc sql2 sql { code2 [list db eval $sql] } + + code1 { sqlite3 db test.db } + code2 { sqlite3 db test.db } + + set bintag [string map {testfixture {}} $bin] + set bintag [string map {\.exe {}} $bintag] + if {$bintag == ""} {set bintag self} + set saved_prefix $::testprefix + append ::testprefix ".$bintag" + + uplevel $script + + set ::testprefix $saved_prefix + + catch { code1 { db close } } + catch { code2 { db close } } + catch { close $::bc_chan } +} + +proc do_all_bc_test {script} { + foreach bin $::BC(binaries) { + uplevel [list do_bc_test $bin $script] + } +} diff --git a/test/check.test b/test/check.test index d2867a096e..4d1a44117d 100644 --- a/test/check.test +++ b/test/check.test @@ -117,9 +117,9 @@ do_test check-1.17 { do_test check-2.1 { execsql { CREATE TABLE t2( - x INTEGER CHECK( typeof(coalesce(x,0))=="integer" ), - y REAL CHECK( typeof(coalesce(y,0.1))=='real' ), - z TEXT CHECK( typeof(coalesce(z,''))=='text' ) + x INTEGER CONSTRAINT one CHECK( typeof(coalesce(x,0))=="integer" ), + y REAL CONSTRAINT two CHECK( typeof(coalesce(y,0.1))=='real' ), + z TEXT CONSTRAINT three CHECK( typeof(coalesce(z,''))=='text' ) ); } } {} @@ -141,17 +141,17 @@ do_test check-2.4 { catchsql { INSERT INTO t2 VALUES(1.1, NULL, NULL); } -} {1 {constraint failed}} +} {1 {constraint one failed}} do_test check-2.5 { catchsql { INSERT INTO t2 VALUES(NULL, 5, NULL); } -} {1 {constraint failed}} +} {1 {constraint two failed}} do_test check-2.6 { catchsql { INSERT INTO t2 VALUES(NULL, NULL, 3.14159); } -} {1 {constraint failed}} +} {1 {constraint three failed}} ifcapable subquery { do_test check-3.1 { diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 4dfa9b8ed2..b2ec156655 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# Tests for the sqlite3_stmt_status() function +# Tests for the sqlite3_db_status() function # set testdir [file dirname $argv0] @@ -33,6 +33,10 @@ proc db_hit_miss {db {reset 0}} { list $nHit $nMiss } +proc db_write {db {reset 0}} { + sqlite3_db_status $db CACHE_WRITE $reset +} + do_test 1.1 { db close sqlite3 db test.db @@ -72,5 +76,24 @@ do_test 1.7 { do_test 1.8 { sqlite3_db_status db CACHE_HIT 0 } {0 2 0} do_test 1.9 { sqlite3_db_status db CACHE_MISS 0 } {0 1 0} +do_test 2.1 { db_write db } {0 0 0} +do_test 2.2 { + execsql { INSERT INTO t1 VALUES(4, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.3 { db_write db 1 } {0 4 0} +do_test 2.4 { db_write db 0 } {0 0 0} +do_test 2.5 { db_write db 1 } {0 0 0} + +do_test 2.6 { + execsql { PRAGMA journal_mode = WAL } + db_write db 1 +} {0 1 0} +do_test 2.7 { + execsql { INSERT INTO t1 VALUES(5, randomblob(600)) } + db_write db +} {0 4 0} +do_test 2.8 { db_write db 1 } {0 4 0} +do_test 2.9 { db_write db 0 } {0 0 0} finish_test diff --git a/test/e_insert.test b/test/e_insert.test index a10d5283de..ac4361f8df 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -50,7 +50,7 @@ proc do_insert_tests {args} { uplevel do_select_tests $args } -# EVIDENCE-OF: R-55375-41353 -- syntax diagram insert-stmt +# EVIDENCE-OF: R-21350-31508 -- syntax diagram insert-stmt # do_insert_tests e_insert-0 { 1 "INSERT INTO a1 DEFAULT VALUES" {} @@ -123,6 +123,20 @@ do_insert_tests e_insert-0 { 68 "INSERT OR IGNORE INTO a1 (b, a) SELECT c, b FROM a2" {} 69 "REPLACE INTO a1 (b, a) SELECT c, b FROM a2" {} 70 "REPLACE INTO main.a1 (b, a) SELECT c, b FROM a2" {} + 71 "INSERT INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 72 "INSERT INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 73 "INSERT OR ROLLBACK INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 74 "INSERT OR ROLLBACK INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 75 "INSERT OR ABORT INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 76 "INSERT OR ABORT INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 77 "INSERT OR REPLACE INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 78 "INSERT OR REPLACE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 79 "INSERT OR FAIL INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 80 "INSERT OR FAIL INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 81 "INSERT OR FAIL INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} + 82 "INSERT OR IGNORE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 83 "REPLACE INTO a1 (b, a) VALUES(1, 2),(3,4)" {} + 84 "REPLACE INTO main.a1 (b, a) VALUES(1, 2),(3,4)" {} } delete_all_data diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index 4d4ae38bae..2ed1f70bf6 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -14,6 +14,129 @@ # to use Tcl. # +#------------------------------------------------------------------------- +# INSTRUCTIONS +# +# The following commands are available: +# +# fts3_build_db_1 N +# Using database handle [db] create an FTS4 table named t1 and populate +# it with N rows of data. N must be less than 10,000. Refer to the +# header comments above the proc implementation below for details. +# +# fts3_build_db_2 N +# Using database handle [db] create an FTS4 table named t2 and populate +# it with N rows of data. N must be less than 100,000. Refer to the +# header comments above the proc implementation below for details. +# +# fts3_integrity_check TBL +# TBL must be an FTS table in the database currently opened by handle +# [db]. This proc loads and tokenizes all documents within the table, +# then checks that the current contents of the FTS index matches the +# results. +# +# fts3_terms TBL WHERE +# Todo. +# +# fts3_doclist TBL TERM WHERE +# Todo. +# +# +# + +#------------------------------------------------------------------------- +# USAGE: fts3_build_db_1 SWITCHES N +# +# Build a sample FTS table in the database opened by database connection +# [db]. The name of the new table is "t1". +# +proc fts3_build_db_1 {args} { + + set default(-module) fts4 + + set nArg [llength $args] + if {($nArg%2)==0} { + error "wrong # args: should be \"fts3_build_db_1 ?switches? n\"" + } + + set n [lindex $args [expr $nArg-1]] + array set opts [array get default] + array set opts [lrange $args 0 [expr $nArg-2]] + foreach k [array names opts] { + if {0==[info exists default($k)]} { error "unknown option: $k" } + } + + if {$n > 10000} {error "n must be <= 10000"} + db eval "CREATE VIRTUAL TABLE t1 USING $opts(-module) (x, y)" + + set xwords [list zero one two three four five six seven eight nine ten] + set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] + + for {set i 0} {$i < $n} {incr i} { + set x "" + set y "" + + set x [list] + lappend x [lindex $xwords [expr ($i / 1000) % 10]] + lappend x [lindex $xwords [expr ($i / 100) % 10]] + lappend x [lindex $xwords [expr ($i / 10) % 10]] + lappend x [lindex $xwords [expr ($i / 1) % 10]] + + set y [list] + lappend y [lindex $ywords [expr ($i / 1000) % 10]] + lappend y [lindex $ywords [expr ($i / 100) % 10]] + lappend y [lindex $ywords [expr ($i / 10) % 10]] + lappend y [lindex $ywords [expr ($i / 1) % 10]] + + db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) } + } +} + +#------------------------------------------------------------------------- +# USAGE: fts3_build_db_2 N ARGS +# +# Build a sample FTS table in the database opened by database connection +# [db]. The name of the new table is "t2". +# +proc fts3_build_db_2 {args} { + + set default(-module) fts4 + set default(-extra) "" + + set nArg [llength $args] + if {($nArg%2)==0} { + error "wrong # args: should be \"fts3_build_db_1 ?switches? n\"" + } + + set n [lindex $args [expr $nArg-1]] + array set opts [array get default] + array set opts [lrange $args 0 [expr $nArg-2]] + foreach k [array names opts] { + if {0==[info exists default($k)]} { error "unknown option: $k" } + } + + if {$n > 100000} {error "n must be <= 100000"} + + set sql "CREATE VIRTUAL TABLE t2 USING $opts(-module) (content" + if {$opts(-extra) != ""} { + append sql ", " $opts(-extra) + } + append sql ")" + db eval $sql + + set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z ""] + + for {set i 0} {$i < $n} {incr i} { + set word "" + set nChar [llength $chars] + append word [lindex $chars [expr {($i / 1) % $nChar}]] + append word [lindex $chars [expr {($i / $nChar) % $nChar}]] + append word [lindex $chars [expr {($i / ($nChar*$nChar)) % $nChar}]] + + db eval { INSERT INTO t2(docid, content) VALUES($i, $word) } + } +} + #------------------------------------------------------------------------- # USAGE: fts3_integrity_check TBL # @@ -46,6 +169,7 @@ proc fts3_integrity_check {tbl} { fts3_read2 $tbl 1 A foreach zTerm [array names A] { + #puts $zTerm foreach doclist $A($zTerm) { set docid 0 while {[string length $doclist]>0} { @@ -97,7 +221,7 @@ proc fts3_integrity_check {tbl} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing" lappend errors $es } else { - if {$C($iDoc,$iCol,$pos) != "$term"} { + if {[string compare $C($iDoc,$iCol,$pos) $term]} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index " append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\"" lappend errors $es @@ -233,7 +357,8 @@ proc fts3_readleaf {blob} { set zTerm [string range $zPrev 0 [expr $nPrefix-1]] append zTerm [gobble_string blob $nSuffix] - set doclist [gobble_string blob [gobble_varint blob]] + set nDoclist [gobble_varint blob] + set doclist [gobble_string blob $nDoclist] lappend terms $zTerm $doclist set zPrev $zTerm @@ -249,7 +374,9 @@ proc fts3_read2 {tbl where varname} { FROM ${tbl}_segdir WHERE $where ORDER BY level ASC, idx DESC " { - if {$start_block == 0} { + set c 0 + binary scan $root c c + if {$c==0} { foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } } else { db eval " SELECT block @@ -258,7 +385,6 @@ proc fts3_read2 {tbl where varname} { ORDER BY blockid " { foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } - } } } diff --git a/test/fts4check.test b/test/fts4check.test new file mode 100644 index 0000000000..cc1d018aad --- /dev/null +++ b/test/fts4check.test @@ -0,0 +1,155 @@ +# 2012 March 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS 'integrity-check' function, +# used to check if the current FTS index accurately reflects the content +# of the table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4check + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +# Run the integrity-check on FTS table $tbl using database handle $db. If +# the integrity-check passes, return "ok". Otherwise, throw an exception. +# +proc fts_integrity {db tbl} { + $db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')" + return "ok" +} + +#------------------------------------------------------------------------- +# Test cases 1.* +# +# 1.0: Build a reasonably sized FTS table (5000 rows). +# +# 1.1: Run the integrity check code to check it passes. +# +# 1.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# + +# Build an FTS table and check the integrity-check passes. +# +do_test 1.0 { fts3_build_db_1 5000 } {} +do_test 1.1 { fts_integrity db t1 } {ok} + +# Mess around with the underlying tables. Check that this causes the +# integrity-check test to fail. +# +foreach {tn disruption} { + 1 { + INSERT INTO t1_content(docid, c0x, c1y) VALUES(NULL, 'a', 'b'); + } + 2 { + DELETE FROM t1_content WHERE docid = (SELECT max(docid) FROM t1_content); + } + 3 { + DELETE FROM t1_segdir WHERE level=0 AND idx=( + SELECT max(idx) FROM t1_segdir WHERE level=0 + ); + } +} { + do_execsql_test 1.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 1.2.2.$tn { + INSERT INTO t1 (t1) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 1.2.3.$tn "ROLLBACK" +} + +do_test 1.3 { fts_integrity db t1 } {ok} + +#------------------------------------------------------------------------- +# Test cases 2.* +# +# 2.0: Build a reasonably sized FTS table (20000 rows) that includes +# prefix indexes. +# +# 2.1: Run the integrity check code to check it passes. +# +# 2.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# + +do_test 2.0 { fts3_build_db_2 -extra {prefix="3,1"} 20000 } {} +do_test 2.1 { fts_integrity db t2 } {ok} +foreach {tn disruption} { + 1 { + INSERT INTO t2_content VALUES(NULL, 'xyz') + } + 3 { + DELETE FROM t2_segdir WHERE level=0 AND idx=( + SELECT max(idx) FROM t2_segdir WHERE level=1024 + ); + } +} { + do_execsql_test 2.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 2.2.2.$tn { + INSERT INTO t2 (t2) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 2.2.3.$tn "ROLLBACK" +} + + +#------------------------------------------------------------------------- +# Test cases 3.* +# +# 3.0: Build a reasonably sized FTS table (5000 rows) that includes +# prefix indexes and uses the languageid= feature. +# +# 3.1: Run the integrity check code to check it passes. +# +# 3.2: Make a series of minor changes to the underlying FTS data structures +# (e.g. delete or insert a row from the %_content table). Check that +# this causes the integrity-check code to fail. +# +do_test 3.0 { + reset_db + fts3_build_db_1 5000 + execsql { + CREATE VIRTUAL TABLE t3 USING fts4(x, y, prefix="2,3", languageid=langid); + } + foreach docid [execsql {SELECT docid FROM t1 ORDER BY 1 ASC}] { + execsql { + INSERT INTO t3(x, y, langid) + SELECT x, y, (docid%9)*4 FROM t1 WHERE docid=$docid; + } + } +} {} +do_test 3.1 { fts_integrity db t3 } {ok} + +foreach {tn disruption} { + 1 { + INSERT INTO t3_content(c0x, c1y, langid) VALUES(NULL, 'a', 0); + } + 2 { + UPDATE t3_content SET langid=langid+1 WHERE rowid = ( + SELECT max(rowid) FROM t3_content + ) + } +} { + do_execsql_test 3.2.1.$tn "BEGIN; $disruption" + do_catchsql_test 3.2.2.$tn { + INSERT INTO t3 (t3) VALUES('integrity-check') + } {1 {database disk image is malformed}} + do_execsql_test 3.2.3.$tn "ROLLBACK" +} + +finish_test diff --git a/test/fts4langid.test b/test/fts4langid.test index 8ad06766b7..843e11f9ab 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -472,15 +472,14 @@ foreach lid [list 4 [expr 1<<30]] { } {8 0} do_execsql_test 5.4.$lid.4 { - INSERT INTO t6(t6) VALUES('optimize'); + INSERT INTO t6(t6) VALUES('merge=100,3'); + INSERT INTO t6(t6) VALUES('merge=100,3'); SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; } {1 2 5} do_execsql_test 5.4.$lid.5 { SELECT count(*) FROM t6_segdir; SELECT count(*) FROM t6_segments; - } {1 0} + } {4 4} } - - finish_test diff --git a/test/fts4merge.test b/test/fts4merge.test new file mode 100644 index 0000000000..fabb651e64 --- /dev/null +++ b/test/fts4merge.test @@ -0,0 +1,341 @@ +# 2012 March 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the incremental merge function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +proc fts3_integrity_check {tbl} { + db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')" + return "ok" +} + +foreach mod {fts3 fts4} { + set ::testprefix fts4merge-$mod + reset_db + + #------------------------------------------------------------------------- + # Test cases 1.* + # + do_test 1.0 { fts3_build_db_1 -module $mod 1004 } {} + do_test 1.1 { fts3_integrity_check t1 } {ok} + do_execsql_test 1.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6 7 8 9 10 11} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} + } + + for {set i 0} {$i<20} {incr i} { + do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') } + do_test 1.2.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.2.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} + } + + do_execsql_test 1.3 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3} + 1 {0 1 2 3 4 5 6} + 2 {0 1 2 3} + } + + for {set i 0} {$i<100} {incr i} { + do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') } + do_test 1.4.$i.2 { fts3_integrity_check t1 } ok + do_execsql_test 1.4.$i.3 { + SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' + } {123 132 213 231 312 321} + } + + do_execsql_test 1.5 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 2 {0 1} + 3 0 + } + + #------------------------------------------------------------------------- + # Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are + # handled correctly. + # + do_execsql_test 2.0 "CREATE VIRTUAL TABLE t2 USING $mod" + + foreach {tn arg} { + 1 {merge=abc} + 2 {merge=%%%} + 3 {merge=,} + 4 {merge=5,} + 5 {merge=6,%} + 6 {merge=6,six} + 7 {merge=6,1} + 8 {merge=6,0} + } { + do_catchsql_test 2.$tn { + INSERT INTO t2(t2) VALUES($arg); + } {1 {SQL logic error or missing database}} + } + + #------------------------------------------------------------------------- + # Test cases 3.* + # + do_test 3.0 { + reset_db + execsql { PRAGMA page_size = 512 } + fts3_build_db_2 -module $mod 30040 + } {} + do_test 3.1 { fts3_integrity_check t2 } {ok} + + do_execsql_test 3.2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6} + 1 {0 1 2 3 4} + 2 {0 1 2 3 4} + 3 {0 1 2 3 4 5 6} + } + + do_execsql_test 3.3 { + INSERT INTO t2(t2) VALUES('merge=1000000,2'); + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level + } { + 0 0 + 2 0 + 3 0 + 4 0 + 6 0 + } + + #------------------------------------------------------------------------- + # Test cases 4.* + # + reset_db + do_execsql_test 4.1 " + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t4 USING $mod; + PRAGMA main.page_size; + " {512} + + do_test 4.2 { + foreach x {a c b d e f g h i j k l m n o p} { + execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" + } + execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} + } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} + + foreach {tn expect} { + 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" + 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" + 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" + 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" + 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" + 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" + 7 "0 {0 1 2 3 4 5 6 7} 1 0" + 8 "0 {0 1 2 3 4 5 6} 1 0" + 9 "0 {0 1 2 3 4 5} 1 0" + } { + do_execsql_test 4.3.$tn { + INSERT INTO t4(t4) VALUES('merge=1,16'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; + } $expect + } + + do_execsql_test 4.4.1 { + SELECT quote(value) FROM t4_stat WHERE rowid=1 + } {X'0006'} + + do_execsql_test 4.4.2 { + DELETE FROM t4_stat WHERE rowid=1; + INSERT INTO t4(t4) VALUES('merge=1,12'); + SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; + } "0 {0 1 2 3 4 5} 1 0" + + + #------------------------------------------------------------------------- + # Test cases 5.* + # + # Test that if a crisis-merge occurs that disrupts an ongoing incremental + # merge, the next call to "merge=A,B" identifies this and starts a new + # incremental merge. There are two scenarios: + # + # * There are less segments on the input level that the disrupted + # incremental merge operated on, or + # + # * Sufficient segments exist on the input level but the segments + # contain keys smaller than the largest key in the potential output + # segment. + # + do_test 5.1 { + reset_db + fts3_build_db_1 -module $mod 1000 + } {} + + do_execsql_test 5.2 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + } { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} + } + + do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('merge=1,5'); + INSERT INTO t1(t1) VALUES('merge=1,5'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + } { + 0 {0 1 2} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} + 2 {0 1 2 3} + } + + do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} + do_test 5.5 { + foreach docid [execsql {SELECT docid FROM t1}] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } + } {} + + do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} + + do_execsql_test 5.7 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 {0 1 2 3 4 5 6 7 8 9 10} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} + 2 {0 1 2 3 4 5 6 7} + X'0105' + } + + do_execsql_test 5.8 { + INSERT INTO t1(t1) VALUES('merge=1,6'); + INSERT INTO t1(t1) VALUES('merge=1,6'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 {0 1 2 3 4} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3 4 5 6 7 8} X'0106' + } + + do_test 5.8.1 { fts3_integrity_check t1 } ok + + do_test 5.9 { + set L [expr 16*16*7 + 16*3 + 12] + foreach docid [execsql { + SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L + }] { + execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} + } + } {} + + do_execsql_test 5.10 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 0 1 {0 1} 2 0 3 0 X'0106' + } + + do_execsql_test 5.11 { + INSERT INTO t1(t1) VALUES('merge=1,6'); + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; + SELECT quote(value) from t1_stat WHERE rowid=1; + } { + 0 0 1 {0 1} 2 0 3 0 X'' + } + + #------------------------------------------------------------------------- + # Test cases 6.* + # + # At one point the following test caused an assert() to fail (because the + # second 'merge=1,2' operation below actually "merges" a single input + # segment, which was unexpected). + # + do_test 6.1 { + reset_db + set a [string repeat a 900] + set b [string repeat b 900] + set c [string repeat c 900] + set d [string repeat d 900] + + execsql "CREATE VIRTUAL TABLE t1 USING $mod" + execsql { + BEGIN; + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + COMMIT; + } + + execsql { + INSERT INTO t1(t1) VALUES('merge=1,2'); + INSERT INTO t1(t1) VALUES('merge=1,2'); + } + } {} + + #------------------------------------------------------------------------- + # Test cases 7.* + # + # Test that the value returned by sqlite3_total_changes() increases by + # 1 following a no-op "merge=A,B", or by more than 1 if actual work is + # performed. + # + do_test 7.0 { + reset_db + fts3_build_db_1 -module $mod 1000 + } {} + + do_execsql_test 7.1 { + SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level + } { + 0 {0 1 2 3 4 5 6 7} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2} + } + do_test 7.2 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=2,10') } + expr { ([db total_changes] - $x)>1 } + } {1} + do_test 7.3 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {1} + do_test 7.4 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {0} + do_test 7.5 { + set x [db total_changes] + execsql { INSERT INTO t1(t1) VALUES('merge=200,10') } + expr { ([db total_changes] - $x)>1 } + } {0} + +} + +finish_test diff --git a/test/fts4merge2.test b/test/fts4merge2.test new file mode 100644 index 0000000000..308b6929fa --- /dev/null +++ b/test/fts4merge2.test @@ -0,0 +1,38 @@ + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl +set ::testprefix fts4merge2 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_test 1.0 { + fts3_build_db_1 1000 + faultsim_save_and_close +} {} + +do_faultsim_test 1.1 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t1(t1) VALUES('merge=32,4') } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 1.2 -faults oom-t* -prep { + if {$iFail<100} {set iFail 803} + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t1(t1) VALUES('merge=1,2') } + execsql { INSERT INTO t1(t1) VALUES('merge=1,2') } +} -test { + faultsim_test_result {0 {}} +} + +finish_test diff --git a/test/fts4merge3.test b/test/fts4merge3.test new file mode 100644 index 0000000000..d591329fb0 --- /dev/null +++ b/test/fts4merge3.test @@ -0,0 +1,97 @@ +# 2012 March 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the incremental merge function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +source $testdir/lock_common.tcl +source $testdir/bc_common.tcl + +set ::testprefix fts4merge3 + +if {"" == [bc_find_binaries backcompat.test]} { + finish_test + return +} + +db close +do_all_bc_test { + + sql2 { PRAGMA page_size = 512 } + if { 0==[catch { sql2 { CREATE VIRTUAL TABLE x USING fts4 } } ] } { + + # Build a large database. + set msg "this takes around 12 seconds" + do_test "1.1 ($msg)" { fts3_build_db_2 20000 } {} + + # Run some queries on it, using the old and new versions. + do_test 1.2 { sql1 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485} + do_test 1.3 { sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485} + + do_test 1.4 { sql2 "PRAGMA page_count" } {1286} + do_test 1.5 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } [list 0 15 1 1 2 14 3 4] + + # Run some incr-merge operations on the db. + for {set i 0} {$i<10} {incr i} { + do_test 1.6.$i.1 { sql1 { INSERT INTO t2(t2) VALUES('merge=2,2') } } {} + do_test 1.6.$i.2 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485} + } + + do_test 1.7 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } [list 0 1 2 18 3 5] + + # Using the old connection, insert many rows. + do_test 1.8 { + for {set i 0} {$i < 1500} {incr i} { + sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i" + } + } {} + + do_test 1.9 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } [list 0 13 1 13 2 5 3 6] + + # Run a big incr-merge operation on the db. + do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {} + do_test 1.11 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485} + + do_test 1.12 { + for {set i 0} {$i < 1500} {incr i} { + sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i" + } + } {} + do_test 1.13 { + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485 22985} + + do_test 1.14 { + sql2 "INSERT INTO t2(t2) VALUES('optimize')" + sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" + } {1485 21485 22985} + + do_test 1.15 { sql2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1 + } } {6 1} + } +} + + +finish_test diff --git a/test/func.test b/test/func.test index eef0543986..ba1ea026d7 100644 --- a/test/func.test +++ b/test/func.test @@ -1247,4 +1247,48 @@ do_test func-28.1 { } } {1 {unknown function: nosuchfunc()}} +# Verify that the length() and typeof() functions do not actually load +# the content of their argument. +# +do_test func-29.1 { + db eval { + CREATE TABLE t29(id INTEGER PRIMARY KEY, x, y); + INSERT INTO t29 VALUES(1, 2, 3), (2, NULL, 4), (3, 4.5, 5); + INSERT INTO t29 VALUES(4, randomblob(1000000), 6); + INSERT INTO t29 VALUES(5, "hello", 7); + } + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(x), length(x), typeof(y) FROM t29 ORDER BY id} +} {integer 1 integer null {} integer real 3 integer blob 1000000 integer text 5 integer} +do_test func-29.2 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} +do_test func-29.3 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(+x) FROM t29 ORDER BY id} +} {integer null real blob text} +do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x +} {many} +do_test func-29.5 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT sum(length(x)) FROM t29} +} {1000009} +do_test func-29.6 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} + + finish_test diff --git a/test/incrblob4.test b/test/incrblob4.test new file mode 100644 index 0000000000..a96356b06f --- /dev/null +++ b/test/incrblob4.test @@ -0,0 +1,90 @@ +# 2012 March 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable {!incrblob} { finish_test ; return } +set testprefix incrblob4 + +proc create_t1 {} { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(k INTEGER PRIMARY KEY, v); + } +} + +proc populate_t1 {} { + set data [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + foreach d $data { + set blob [string repeat $d 900] + execsql { INSERT INTO t1(v) VALUES($blob) } + } +} + + +do_test 1.1 { + create_t1 + populate_t1 +} {} + +do_test 1.2 { + set blob [db incrblob t1 v 5] + read $blob 10 +} {eeeeeeeeee} + +do_test 1.3 { + execsql { DELETE FROM t1 } + populate_t1 +} {} + + + +do_test 2.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 2.2 { + set blob [db incrblob t1 v 10] + read $blob 10 +} {jjjjjjjjjj} + +do_test 2.3 { + set new [string repeat % 900] + execsql { DELETE FROM t1 WHERE k=10 } + execsql { DELETE FROM t1 WHERE k=9 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + + + +do_test 3.1 { + reset_db + create_t1 + populate_t1 +} {} + +do_test 3.2 { + set blob [db incrblob t1 v 20] + read $blob 10 +} {tttttttttt} + +do_test 3.3 { + set new [string repeat % 900] + execsql { UPDATE t1 SET v = $new WHERE k = 20 } + execsql { DELETE FROM t1 WHERE k=19 } + execsql { INSERT INTO t1(v) VALUES($new) } +} {} + +finish_test + diff --git a/test/pager1.test b/test/pager1.test index 45253857c3..2ac56dec8c 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -1768,7 +1768,7 @@ do_test pager1-18.2 { catchsql { SELECT count(*) FROM t1 } db2 } {1 {database disk image is malformed}} db2 close -do_test pager1-18.3 { +do_test pager1-18.3.1 { execsql { CREATE TABLE t2(x); INSERT INTO t2 VALUES(a_string(5000)); @@ -1776,13 +1776,38 @@ do_test pager1-18.3 { set pgno [expr ([file size test.db] / 1024)-2] hexio_write test.db [expr ($pgno-1)*1024] 00000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + # even though x is malformed, because typeof() does + # not load the content of x, the error is not noticed. + catchsql { SELECT typeof(x) FROM t2 } db2 +} {0 text} +do_test pager1-18.3.2 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 +} {1 {database disk image is malformed}} +db2 close +do_test pager1-18.3.3 { + execsql { + DELETE FROM t2; + INSERT INTO t2 VALUES(randomblob(5000)); + } + set pgno [expr ([file size test.db] / 1024)-2] + hexio_write test.db [expr ($pgno-1)*1024] 00000000 + sqlite3 db2 test.db + # even though x is malformed, because length() and typeof() do + # not load the content of x, the error is not noticed. + catchsql { SELECT length(x), typeof(x) FROM t2 } db2 +} {0 {5000 blob}} +do_test pager1-18.3.4 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.4 { hexio_write test.db [expr ($pgno-1)*1024] 90000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.5 { diff --git a/test/permutations.test b/test/permutations.test index d7ce9cfe3d..1e8d98986b 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -187,7 +187,8 @@ test_suite "fts3" -prefix "" -description { fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test - fts3corrupt2.test fts3first.test fts4langid.test + fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test + fts4check.test } diff --git a/test/trace2.test b/test/trace2.test index 0e3130ec00..2f7ae7d5a2 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -130,11 +130,11 @@ ifcapable fts3 { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" - "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" - "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" + "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" + "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" - "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } do_trace_test 2.3 { @@ -146,7 +146,7 @@ ifcapable fts3 { "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" - "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } diff --git a/test/vtab1.test b/test/vtab1.test index f056b2f368..38aec09eae 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix vtab1 ifcapable !vtab||!schema_pragmas { finish_test @@ -43,6 +44,9 @@ ifcapable !vtab||!schema_pragmas { # # vtab1-14.*: Test 'IN' constraints - i.e. "SELECT * FROM t1 WHERE id IN(...)" # +# vtab1-18.*: Check that the LIKE optimization is not applied when the lhs +# is a virtual table column. +# #---------------------------------------------------------------------- @@ -1218,5 +1222,57 @@ do_test vtab1-17.1 { } } {} +#------------------------------------------------------------------------- +# The following tests - vtab1-18.* - test that the optimization of LIKE +# constraints in where.c plays well with virtual tables. +# +# 18.1.*: Case-insensitive LIKE. +# 18.2.*: Case-sensitive LIKE. +# unset -nocomplain echo_module_begin_fail + +do_execsql_test 18.1.0 { + CREATE TABLE t6(a, b TEXT); + CREATE INDEX i6 ON t6(b, a); + INSERT INTO t6 VALUES(1, 'Peter'); + INSERT INTO t6 VALUES(2, 'Andrew'); + INSERT INTO t6 VALUES(3, 'James'); + INSERT INTO t6 VALUES(4, 'John'); + INSERT INTO t6 VALUES(5, 'Phillip'); + INSERT INTO t6 VALUES(6, 'Bartholomew'); + CREATE VIRTUAL TABLE e6 USING echo(t6); +} + +foreach {tn sql res filter} { + 1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5} + {xFilter {SELECT rowid, * FROM 't6' WHERE b > ?} James} + + 1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4} + {xFilter {SELECT rowid, * FROM 't6' WHERE b >= ? AND b < ?} J K} + + 1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} + + 1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} +} { + set echo_module {} + do_execsql_test 18.$tn.1 $sql $res + do_test 18.$tn.2 { lrange $::echo_module 2 end } $filter +} + +do_execsql_test 18.2.0 { PRAGMA case_sensitive_like = ON } +foreach {tn sql res filter} { + 2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4} + {xFilter {SELECT rowid, * FROM 't6'}} + + 2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {} + {xFilter {SELECT rowid, * FROM 't6'}} +} { + set echo_module {} + do_execsql_test 18.$tn.1 $sql $res + do_test 18.$tn.2 { lrange $::echo_module 2 end } $filter +} +do_execsql_test 18.2.x { PRAGMA case_sensitive_like = OFF } + finish_test