diff --git a/Makefile.msc b/Makefile.msc index 3c14cc2087..8e3fed8f8c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -995,9 +995,12 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c clean: del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib + del /Q *.da *.bb *.bbg gmon.out del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* del /Q mkkeywordhash.exe keywordhash.h + -rmdir /Q/S .deps + -rmdir /Q/S .libs -rmdir /Q/S tsrc del /Q .target_source del /Q tclsqlite3.exe diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d17809fe54..96905fc14b 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -286,10 +286,6 @@ ** will eventually overtake the earlier data and knock it out. The ** query logic likewise merges doclists so that newer data knocks out ** older data. -** -** TODO(shess) Provide a VACUUM type operation to clear out all -** deletions and duplications. This would basically be a forced merge -** into a single segment. */ #include "fts3Int.h" @@ -469,6 +465,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -545,7 +542,9 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + const char *zLanguageid; + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Create a list of user columns for the virtual table */ @@ -556,7 +555,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid ); if( !zCols || !zSql ){ rc = SQLITE_NOMEM; @@ -585,6 +585,7 @@ static int fts3CreateTables(Fts3Table *p){ sqlite3 *db = p->db; /* The database connection */ if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; char *zContentCols; /* Columns of %_content table */ /* Create a list of user columns for the content table */ @@ -593,6 +594,9 @@ static int fts3CreateTables(Fts3Table *p){ char *z = p->azColumn[i]; zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } if( zContentCols==0 ) rc = SQLITE_NOMEM; /* Create the content table */ @@ -792,14 +796,20 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } sqlite3_free(zFree); }else{ fts3Appendf(pRc, &zRet, "rowid"); for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } } - fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", p->zDb, (p->zContentTbl ? p->zContentTbl : p->zName), (p->zContentTbl ? "" : "_content") @@ -842,6 +852,9 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } sqlite3_free(zFree); return zRet; } @@ -1057,6 +1070,7 @@ static int fts3InitVtab( char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -1106,7 +1120,8 @@ static int fts3InitVtab( { "compress", 8 }, /* 2 -> COMPRESS */ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 } /* 5 -> CONTENT */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 } /* 6 -> LANGUAGEID */ }; int iOpt; @@ -1160,12 +1175,18 @@ static int fts3InitVtab( bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; - default: /* CONTENT */ - assert( iOpt==5 ); - sqlite3_free(zUncompress); + case 5: /* CONTENT */ + sqlite3_free(zContent); zContent = zVal; zVal = 0; break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -1195,8 +1216,20 @@ static int fts3InitVtab( sqlite3_free((void*)aCol); aCol = 0; rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; j0 ); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1243,7 +1276,9 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bDescIdx = bDescIdx; p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; zContent = 0; + zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1306,6 +1341,7 @@ fts3_init_out: sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free(zContent); + sqlite3_free(zLanguageid); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1357,6 +1393,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts3Table *p = (Fts3Table *)pVTab; int i; /* Iterator variable */ int iCons = -1; /* Index of constraint to use */ + int iLangidCons = -1; /* Index of langid=x constraint, if present */ /* By default use a full table scan. This is an expensive option, ** so search through the constraints to see if a more efficient @@ -1369,7 +1406,8 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ if( pCons->usable==0 ) continue; /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + if( iCons<0 + && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) ){ pInfo->idxNum = FTS3_DOCID_SEARCH; @@ -1392,7 +1430,13 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; pInfo->estimatedCost = 2.0; iCons = i; - break; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; } } @@ -1400,6 +1444,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } + if( iLangidCons>=0 ){ + pInfo->aConstraintUsage[iLangidCons].argvIndex = 2; + } /* Regardless of the strategy selected, FTS can deliver rows in rowid (or ** docid) order. Both ascending and descending are possible. @@ -2549,6 +2596,7 @@ static int fts3SegReaderCursorAppend( */ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2577,7 +2625,7 @@ static int fts3SegReaderCursor( if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ @@ -2622,6 +2670,7 @@ static int fts3SegReaderCursor( */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2646,7 +2695,7 @@ int sqlite3Fts3SegReaderCursor( memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( - p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } @@ -2658,11 +2707,14 @@ int sqlite3Fts3SegReaderCursor( */ static int fts3SegReaderCursorAddZero( Fts3Table *p, /* FTS virtual table handle */ + int iLangid, const char *zTerm, /* Term to scan doclist of */ int nTerm, /* Number of bytes in zTerm */ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ - return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); + return fts3SegReaderCursor(p, + iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr + ); } /* @@ -2698,8 +2750,9 @@ static int fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); pSegcsr->bLookup = 1; } } @@ -2707,19 +2760,21 @@ static int fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm+1 ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr ); if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); } } } } if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor( - p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); pSegcsr->bLookup = !isPrefix; } @@ -2874,7 +2929,7 @@ static int fts3FilterMethod( UNUSED_PARAMETER(nVal); assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( nVal==0 || nVal==1 ); + assert( nVal==0 || nVal==1 || nVal==2 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); assert( p->pSegments==0 ); @@ -2899,8 +2954,11 @@ static int fts3FilterMethod( return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, - p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + pCsr->iLangid = 0; + if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -2971,10 +3029,17 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column */ static int fts3ColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ int rc = SQLITE_OK; /* Return Code */ @@ -2982,22 +3047,34 @@ static int fts3ColumnMethod( Fts3Table *p = (Fts3Table *)pCursor->pVtab; /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+1 ); + assert( iCol>=0 && iCol<=p->nColumn+2 ); if( iCol==p->nColumn+1 ){ /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_result_int64(pContext, pCsr->iPrevId); + sqlite3_result_int64(pCtx, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. - */ - sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ - sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); + + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } } } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 16c3de0578..393cd6aea1 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -192,11 +192,12 @@ struct Fts3Table { char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[27]; + sqlite3_stmt *aStmt[28]; char *zReadExprlist; char *zWriteExprlist; @@ -211,12 +212,12 @@ struct Fts3Table { /* TODO: Fix the first paragraph of this comment. ** - ** The following hash table is used to buffer pending index updates during - ** transactions. Variable nPendingData estimates the memory size of the - ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds nMaxPendingData, the buffer is flushed - ** automatically. Variable iPrevDocid is the docid of the most recently - ** inserted record. + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. Variable nPendingData estimates the memory + ** size of the pending data, including hash table overhead, not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, the buffer + ** is flushed automatically. Variable iPrevDocid is the docid of the most + ** recently inserted record. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the @@ -231,12 +232,13 @@ struct Fts3Table { int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These - ** values do not contribution to the FTS computation; they are used for - ** verifying the SQLite core. + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. */ int inTransaction; /* True after xBegin but before xCommit/xRollback */ int mxSavepoint; /* Largest valid xSavepoint integer */ @@ -255,6 +257,7 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ @@ -406,7 +409,7 @@ int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); @@ -427,8 +430,8 @@ int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); +int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -495,7 +498,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ -int sqlite3Fts3ExprParse(sqlite3_tokenizer *, +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, char **, int, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); @@ -504,6 +507,10 @@ int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); int sqlite3Fts3InitTerm(sqlite3 *db); #endif +int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index ada85d796b..89bf3ebf6e 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -376,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 1c3a79071c..a6e3492242 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -92,6 +92,7 @@ int sqlite3_fts3_enable_parentheses = 0; typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ const char **azCol; /* Array of column names for fts3 table */ int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ @@ -127,6 +128,33 @@ static void *fts3MallocZero(int nByte){ return pRet; } +int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + /* ** Extract the next token from buffer z (length n) using the tokenizer @@ -154,15 +182,13 @@ static int getNextToken( Fts3Expr *pRet = 0; int nConsumed = 0; - rc = pModule->xOpen(pTokenizer, z, n, &pCursor); + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken, iStart, iEnd, iPosition; int nByte; /* total space to allocate */ - pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); @@ -268,10 +294,10 @@ static int getNextString( ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase ** structures. */ - rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; - pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ const char *zByte; int nByte, iBegin, iEnd, iPos; @@ -745,6 +771,7 @@ exprparse_out: */ int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ char **azCol, /* Array of column names for fts3 table */ int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ @@ -755,11 +782,13 @@ int sqlite3Fts3ExprParse( int nParsed; int rc; ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; sParse.azCol = (const char **)azCol; sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; - sParse.nNest = 0; sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; @@ -950,7 +979,7 @@ static void fts3ExprTest( } rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 23ef25c5d4..fd5bc9786b 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -532,6 +532,7 @@ static int fts3StringAppend( */ static int fts3SnippetShift( Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ int nSnippet, /* Number of tokens desired for snippet */ const char *zDoc, /* Document text to extract snippet from */ int nDoc, /* Size of buffer zDoc in bytes */ @@ -567,11 +568,10 @@ static int fts3SnippetShift( /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) ** or more tokens in zDoc/nDoc. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); @@ -631,11 +631,10 @@ static int fts3SnippetText( /* Open a token cursor on the document. */ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK ){ int iBegin; /* Offset in zDoc of start of token */ @@ -657,7 +656,9 @@ static int fts3SnippetText( if( !isShiftDone ){ int n = nDoc - iBegin; - rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask); + rc = fts3SnippetShift( + pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); isShiftDone = 1; /* Now that the shift has been done, check if the initial "..." are @@ -1390,9 +1391,10 @@ void sqlite3Fts3Offsets( } /* Initialize a tokenizer iterator to iterate through column iCol. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); if( rc!=SQLITE_OK ) goto offsets_out; - pC->pTokenizer = pTab->pTokenizer; rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); while( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index d3eb690bdd..2108fc1251 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -271,7 +271,7 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 72735f3d12..7cbc9eae9f 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -13,6 +13,9 @@ ** This file is not part of the production FTS code. It is only used for ** testing. It contains a Tcl command that can be used to test if a document ** matches an FTS NEAR expression. +** +** As of March 2012, it also contains a version 1 tokenizer used for testing +** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ #include @@ -314,11 +317,207 @@ static int fts3_configure_incr_load_cmd( return TCL_OK; } +/************************************************************************** +** Beginning of test tokenizer code. +** +** For language 0, this tokenizer is similar to the default 'simple' +** tokenizer. For other languages L, the following: +** +** * Odd numbered languages are case-sensitive. Even numbered +** languages are not. +** +** * Language ids 100 or greater are considered an error. +** +** The implementation assumes that the input contains only ASCII characters +** (i.e. those that may be encoded in UTF-8 using a single byte). +*/ +typedef struct test_tokenizer { + sqlite3_tokenizer base; +} test_tokenizer; + +typedef struct test_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *aInput; /* Input being tokenized */ + int nInput; /* Size of the input in bytes */ + int iInput; /* Current offset in aInput */ + int iToken; /* Index of next token to be returned */ + char *aBuffer; /* Buffer containing current token */ + int nBuffer; /* Number of bytes allocated at pToken */ + int iLangid; /* Configured language id */ +} test_tokenizer_cursor; + +static int testTokenizerCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + test_tokenizer *pNew; + + pNew = sqlite3_malloc(sizeof(test_tokenizer)); + if( !pNew ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(test_tokenizer)); + + *ppTokenizer = (sqlite3_tokenizer *)pNew; + return SQLITE_OK; +} + +static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){ + test_tokenizer *p = (test_tokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +static int testTokenizerOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + int rc = SQLITE_OK; /* Return code */ + test_tokenizer_cursor *pCsr; /* New cursor object */ + + UNUSED_PARAMETER(pTokenizer); + + pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor)); + if( pCsr==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(test_tokenizer_cursor)); + pCsr->aInput = pInput; + if( nBytes<0 ){ + pCsr->nInput = strlen(pInput); + }else{ + pCsr->nInput = nBytes; + } + } + + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return rc; +} + +static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + sqlite3_free(pCsr->aBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int testIsTokenChar(char c){ + return (c>='a' && c<='z') || (c>='A' && c<='Z'); +} +static int testTolower(char c){ + char ret = c; + if( ret>='A' && ret<='Z') ret = ret - ('A'-'a'); + return ret; +} + +static int testTokenizerNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + int rc = SQLITE_OK; + const char *p; + const char *pEnd; + + p = &pCsr->aInput[pCsr->iInput]; + pEnd = &pCsr->aInput[pCsr->nInput]; + + /* Skip past any white-space */ + assert( p<=pEnd ); + while( ppCsr->nBuffer ){ + sqlite3_free(pCsr->aBuffer); + pCsr->aBuffer = sqlite3_malloc(nToken); + } + if( pCsr->aBuffer==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + + if( pCsr->iLangid & 0x00000001 ){ + for(i=0; iaBuffer[i] = pToken[i]; + }else{ + for(i=0; iaBuffer[i] = testTolower(pToken[i]); + } + pCsr->iToken++; + pCsr->iInput = p - pCsr->aInput; + + *ppToken = pCsr->aBuffer; + *pnBytes = nToken; + *piStartOffset = pToken - pCsr->aInput; + *piEndOffset = p - pCsr->aInput; + *piPosition = pCsr->iToken; + } + } + + return rc; +} + +static int testTokenizerLanguage( + sqlite3_tokenizer_cursor *pCursor, + int iLangid +){ + int rc = SQLITE_OK; + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + pCsr->iLangid = iLangid; + if( pCsr->iLangid>=100 ){ + rc = SQLITE_ERROR; + } + return rc; +} + +static int fts3_test_tokenizer_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static const sqlite3_tokenizer_module testTokenizerModule = { + 1, + testTokenizerCreate, + testTokenizerDestroy, + testTokenizerOpen, + testTokenizerClose, + testTokenizerNext, + testTokenizerLanguage + }; + const sqlite3_tokenizer_module *pPtr = &testTokenizerModule; + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj( + (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) + )); + return TCL_OK; +} + +/* +** End of tokenizer code. +**************************************************************************/ + int Sqlitetestfts3_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); Tcl_CreateObjCommand(interp, "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 ); + Tcl_CreateObjCommand( + interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 + ); return TCL_OK; } #endif /* ifdef SQLITE_TEST */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index 6494bb96d8..f6b044ff6a 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -288,11 +288,10 @@ static void testFunc( goto finish; } pTokenizer->pModule = p; - if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ zErr = "error in xOpen()"; goto finish; } - pCsr->pTokenizer = pTokenizer; while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h index 615644506c..c91c7ed790 100644 --- a/ext/fts3/fts3_tokenizer.h +++ b/ext/fts3/fts3_tokenizer.h @@ -52,7 +52,7 @@ typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; struct sqlite3_tokenizer_module { /* - ** Structure version. Should always be set to 0. + ** Structure version. Should always be set to 0 or 1. */ int iVersion; @@ -133,6 +133,15 @@ struct sqlite3_tokenizer_module { int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ int *piPosition /* OUT: Number of tokens returned before this one */ ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); }; struct sqlite3_tokenizer { diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 6d7c3e8d0d..8f97c8be98 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -232,6 +232,8 @@ struct SegmentNode { #define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -285,6 +287,7 @@ static int fts3SqlStmt( /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", }; int rc = SQLITE_OK; @@ -431,6 +434,19 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ return rc; } +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, + int iLangid, + int iIndex, + int iLevel +){ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; +} + + /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -450,6 +466,7 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ int iLevel, /* Level to select */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ @@ -465,14 +482,16 @@ int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); } } *ppStmt = pStmt; @@ -638,6 +657,7 @@ static int fts3PendingTermsAddOne( */ static int fts3PendingTermsAdd( Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ u32 *pnWord /* OUT: Number of tokens inserted */ @@ -667,11 +687,10 @@ static int fts3PendingTermsAdd( return SQLITE_OK; } - rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; } - pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc @@ -714,18 +733,28 @@ static int fts3PendingTermsAdd( ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ -static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ + if( iDocid<=p->iPrevDocid + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; return SQLITE_OK; } @@ -754,11 +783,16 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ ** Argument apVal is the same as the similarly named argument passed to ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ -static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } @@ -779,6 +813,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ ** apVal[p->nColumn+1] Right-most user-defined column ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ @@ -809,9 +844,13 @@ static int fts3InsertData( ** defined columns in the FTS3 table, plus one for the docid field. */ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); - if( rc!=SQLITE_OK ){ - return rc; + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); } + if( rc!=SQLITE_OK ) return rc; /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. @@ -871,6 +910,15 @@ static int fts3DeleteAll(Fts3Table *p, int bContent){ return rc; } +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the @@ -890,16 +938,18 @@ static void fts3DeleteTerms( if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; - for(i=1; i<=p->nColumn; i++){ + int iLangid = langidFromSelect(p, pSelect); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); - if( rc!=SQLITE_OK ){ - sqlite3_reset(pSelect); - *pRC = rc; - return; - } + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]); aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } } rc = sqlite3_reset(pSelect); }else{ @@ -912,7 +962,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int, int); +static int fts3SegmentMerge(Fts3Table *, int, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -931,6 +981,7 @@ static int fts3SegmentMerge(Fts3Table *, int, int); */ static int fts3AllocateSegdirIdx( Fts3Table *p, + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, int *piIdx @@ -939,10 +990,15 @@ static int fts3AllocateSegdirIdx( sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -956,7 +1012,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iIndex, iLevel); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -2179,7 +2235,12 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + int *pnMax +){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndexnIndex ); @@ -2192,8 +2253,10 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int(pStmt, 0); } @@ -2216,6 +2279,7 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ @@ -2243,13 +2307,15 @@ static int fts3DeleteSegdir( if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } @@ -2718,13 +2784,18 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2734,7 +2805,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ assert( iLevel=0 && iIndexnIndex ); - rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ @@ -2746,24 +2817,24 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); bIgnoreEmpty = 1; }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; - rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); - assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevelnIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); @@ -2954,17 +3027,34 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ - int i; int bSeenDone = 0; - int rc = SQLITE_OK; - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - bSeenDone = 1; - rc = SQLITE_OK; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } + sqlite3Fts3SegmentsClose(p); sqlite3Fts3PendingTermsClear(p); @@ -3015,11 +3105,12 @@ static int fts3DoRebuild(Fts3Table *p){ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; - rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0)); + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); aSz[p->nColumn] = 0; for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); } if( p->bHasDocsize ){ @@ -3138,14 +3229,13 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); sqlite3_tokenizer_cursor *pTC = 0; - rc = pModule->xOpen(pT, zText, -1, &pTC); + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ int nToken; /* Number of bytes in token */ int iDum1, iDum2; /* Dummy variables */ int iPos; /* Position of token in zText */ - pTC->pTokenizer = pT; rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; @@ -3245,8 +3335,6 @@ static int fts3DeleteByRowid( rc = fts3DeleteAll(p, 1); *pnDoc = *pnDoc - 1; }else{ - sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); - rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, pRowid, aSzDel); if( p->zContentTbl==0 ){ fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); @@ -3265,7 +3353,16 @@ static int fts3DeleteByRowid( /* ** This function does the work for the xUpdate method of FTS3 virtual -** tables. +** tables. The schema of the virtual table being: +** +** CREATE TABLE ( +** , +**
HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ @@ -3282,6 +3379,10 @@ int sqlite3Fts3UpdateMethod( int bInsertDone = 0; assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); /* Check for a "special" INSERT operation. One of the form: ** @@ -3295,6 +3396,11 @@ int sqlite3Fts3UpdateMethod( goto update_out; } + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ){ @@ -3362,6 +3468,7 @@ int sqlite3Fts3UpdateMethod( /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ @@ -3369,11 +3476,11 @@ int sqlite3Fts3UpdateMethod( } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, *pRowid); + rc = fts3PendingTermsDocid(p, iLangid, *pRowid); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); - rc = fts3InsertTerms(p, apVal, aSzIns); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); } if( p->bHasDocsize ){ fts3InsertDocsize(&rc, p, aSzIns); diff --git a/manifest b/manifest index 408205ca6d..6b587a1394 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Pull\sall\sthe\slatest\strunk\schanges\sinto\sthe\ssessions\sbranch. -D 2012-02-10T17:54:52.604 +C Merge\sin\sthe\slatest\strunk\schanges. +D 2012-03-05T16:26:00.280 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 340b6d1bb4553c389d6837aa437d7c25dc03f980 +F Makefile.msc ceec8f4ac0f7f463f5ec241dc0115de7a84daba2 F Makefile.vxworks 1deb39c8bb047296c30161ffa10c1b5423e632f9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION bb4c2a86abe53ea3c1d6ea515b69a426040e2414 @@ -63,22 +63,22 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4cf7b8e5bbb6667f5d7818fa0bf064fbbb72b086 +F ext/fts3/fts3.c 806632fd0020eed966ab82ea25fe09f1a4c86907 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ce958a6fa92a95462853aa3acc0b69bcda39102f -F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 -F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15 +F ext/fts3/fts3Int.h d1d7f964ddee067bcd16a6af4ba7ecf66220056d +F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 +F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c b7e5276f9f0a5fc7018b6fa55ce0f31f269ef881 -F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a -F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef -F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 -F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 -F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 +F ext/fts3/fts3_snippet.c c9e126c20760988aa7c43c6ea1379db34738282e +F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6 +F ext/fts3/fts3_test.c a026412a41450a014ccb7abdd5efaa7c9711d49e +F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce +F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 1721187a4dec29ef9ae648ad8478da741085af18 +F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -134,11 +134,11 @@ F src/alter.c 149cc80d9257971b0bff34e58fb2263e01998289 F src/analyze.c f32ff304da413851eefa562b04e61ff6cb88248b F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c e9538bad2d4a4fcd4308f1aed7cb18a0fbc968f9 +F src/backup.c 6be23a344d3301ae38e92fddb3a33b91c309fce4 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bb0132a725b4d5ed077924399c4f6d4b9390a721 -F src/btree.h 46e9f04672f1390255bc56865a3238b384d0f2d5 +F src/btree.c 253c3147a4ebbaee42cd329dbdc0856200bbbda7 +F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a @@ -149,7 +149,7 @@ F src/delete.c c6796c89280de6cbad73089adb4cdd01a4d4a1a7 F src/expr.c 00675123e0beec98f999aa4594d2cbe1fec33c1b F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 -F src/func.c d7925f33a8ce2207f86dea5cfb1a4379413312fe +F src/func.c e75f41c421f00762ab9da7dc8fb90af3972cf99f F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -159,10 +159,10 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c 166f9571341f7084172f87e1931f0833778576eb +F src/main.c 18ef52feda1f9e151d1dcd032d0c7b95275a7397 F src/malloc.c 15afac5e59b6584efe072e9933aefb4230e74f97 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c c953f3bfc8fcd31d4de2078697caefeb1dcfd7ff +F src/mem1.c b3677415e69603d6a0e7c5410a1b3731d55beda1 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f @@ -178,25 +178,25 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c d509b369ed376c77bc547961844a105d3907e4fa +F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 -F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a +F src/pager.c 3955b62cdb5bb64559607cb474dd12a6c8e1d4a5 +F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c 350f59843f4ec4fca5dc63d497caf6433096bbdd +F src/pragma.c e708b3bb5704605816f617e0b1d63a5488060715 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 +F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e -F src/sqlite.h.in e805ca92bb66d958423dfcef12879e16bf547209 +F src/sqlite.h.in fc8237a04a17cb31addc670bc9abc99c6712d6d8 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h b62e6f8dfc335f78e89721d54172fa12909e8f2e +F src/sqliteInt.h 3242e4adac818351aed6306a679df3390c771c8b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -218,7 +218,7 @@ F src/test_config.c 576d3d9450fb009ccd8be581eaab7c7e3458cc40 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c f884f6f32e8513d34248d6e1ac8a32047fead254 +F src/test_fuzzer.c 3703a190bd79a43e5f097d59c73ab38961d14872 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -226,7 +226,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee -F src/test_multiplex.c afab2c9d292677ab31e0dd4b3dde2420fb655c5f +F src/test_multiplex.c 0404a61d7795438be5ee5fd3711eed80724df34d F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -242,30 +242,30 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c 07157a0bbfe161cb5e32cad2079abd26cd611c4b -F src/test_vfstrace.c 065c7270a614254b2c68fbc7ba8d1fb1d5cbc823 +F src/test_vfs.c 73f46bd9b5183ebcb77da22773886b81157cdc3d +F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c 89de085a0bf4da448472029d0420a2b1cf1824ee F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 -F src/util.c 9e07bd67dfafe9c75b1da78c87ba030cebbb5388 -F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa -F src/vdbe.c a24b861d9de4e26220c4656b6db22103b7e5f39f +F src/util.c 906731099c4397bf8adf3fa90a833355e7472af0 +F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 +F src/vdbe.c 2591c28c3b11d67a80ff8fca6ceca7c72c140456 F src/vdbe.h 87b8ff40de3f55dbcdc33029416862f517c37a2f F src/vdbeInt.h f1956902b06b4f05ce965aafab6fe220a5477f9c F src/vdbeapi.c 2fc381f651738feb2495cb001cf2114dea596cc3 -F src/vdbeaux.c b7c245fe73ed15090cf626f1250dcb7cdcfb1bd6 +F src/vdbeaux.c 2803275ce14795549fd86a03474cac80f549a569 F src/vdbeblob.c 11248c6362389569764682eb0f59ce910f3cc381 F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843 F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 -F src/wal.c 5f7bcc0610af759953defd769eacebfd98a22bc8 -F src/wal.h eaa00b9a403ddda2b56d01b7afc19ef600f9363f +F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 +F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 +F src/where.c f2cf59751f7facb4c422adf83ddc989aa5772874 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -324,11 +324,11 @@ F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 -F test/cache.test 754baab2f18089fc9bcba7afaeb4dc907c6c6de2 +F test/cache.test f64136b0893c293d0b910ed057b3b711249099a7 F test/capi2.test 835d4cee9f542ea50fa8d01f3fe6de80b0627360 -F test/capi3.test 7200dff6acb17b9a4b6f9918f554eaae04968ddd +F test/capi3.test 9c8b58b6a6aeb14e69bd8c8c7721b47d640464d1 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test ccf0acf045dbacd09f6229aa4efed670aaba76a9 +F test/capi3c.test 1b5424d2ac57b7b443b5de5b9a287642c02279b6 F test/capi3d.test 17b57ca28be3e37e14c2ba8f787d292d84b724a1 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -376,7 +376,7 @@ F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test a18a2ce81add84b17b06559e82ad7bb91bc6ddff -F test/dbstatus.test 3e978f8bdb2362a36a4be63c36a59f542c4cc2a3 +F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b F test/dbstatus2.test dc57b0d9610851c0ff58a8e1b5b191678398b72a F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 @@ -386,28 +386,28 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 76908ed038c5186ffb8acf5954ed64e22056f311 +F test/distinct.test 8c6d12ba53ee8351a5b2d47628acdfad1fc97743 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 48598b15e8fe6554d301e7b65a10c9851f177e84 -F test/e_delete.test ec168cd4b08d681e6d5847f462203755ad647532 +F test/e_delete.test 89aa84d3d1bd284a0689ede04bce10226a5aeaa5 F test/e_droptrigger.test afd5c4d27dec607f5997a66bf7e2498a082cb235 F test/e_dropview.test 583411e470458c5d76148542cfb5a5fa84c8f93e -F test/e_expr.test 4e4399006b3d1ab333721b8e386cabb9fb6d5a89 -F test/e_fkey.test 38039b840ab19331000b0f0eb1d82baa7208a67a +F test/e_expr.test 5489424d3d9a452ac3701cdf4b680ae31a157894 +F test/e_fkey.test 057eed81a41a2b21b1790032f4e8aaba0b2b0e17 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 -F test/e_insert.test 234242b71855af8e8a9b1e141c3533f6d43d8645 +F test/e_insert.test 92d2dab07366aef112f53af4539e30559f5d35a7 F test/e_reindex.test dfedfc32c5a282b0596c6537cbcd4217fbb1a216 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 99202f99a9a3273c6fb0d2e7592b98faeb6c206e +F test/e_select.test f5d4b81205701deacfae42051ae200969c41d2c0 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 -F test/e_update.test dba988a4d079156549a40854074ba4890b0a4577 +F test/e_update.test 161d5dc6a3ed9dd08f5264d13e20735d7a89f00c F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 -F test/e_vacuum.test 5296e25ef871965bac010b9da083dd7e4734526e +F test/e_vacuum.test 331da289ae186656cf5f2eb27f577a89c0c221af F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 796c59832e2b9a52842f382ffda8f3e989db03ad F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test f14fadd76da53405e9885e2431cacf7191d83cdb +F test/eqp.test 6a389bba6ea113fd5179515001be788a38d53ec7 F test/errmsg.test 3bb606db9d040cc6854459f8f5e5a2bcd9b7fd2a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb @@ -508,6 +508,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f +F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -516,14 +517,15 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f +F test/fuzzer1.test 69cf1036b92fd3b8e1fd65bef4d7ee3f085c28fb +F test/fuzzerfault.test ff2282c81797b6a355f0748d8b54c7287c5d2b25 F test/hook.test 94b927b15883f5c1477ab09eecd16275addb08f4 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 -F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d +F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 -F test/incrblob.test 3307c04876fe025e10256e3cc8050ab5a84aa27f +F test/incrblob.test 26fde912a1e0aff158b3a84ef3b265f046aad3be F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 @@ -538,7 +540,7 @@ F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 -F test/insert.test d540650825c98d8082d32f786c611d70e1c21a80 +F test/insert.test 489aa12a027c83d291f5034a83c8c32e6be1dca2 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 87f6798f31d60c4e177622fcc3663367e6ecbd90 @@ -557,10 +559,10 @@ F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe -F test/join6.test bf82cf3f979e9eade83ad0d056a66c5ed71d1901 +F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 8b71ef1ed5798bdc0e6eb616d8694e2c2c188d4d F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 -F test/journal3.test 6fd28532c88b447db844186bc190523108b6dbb4 +F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 9ee3a78f53d52cca737db69293d15dc41c0cbd36 F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa @@ -613,7 +615,7 @@ F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 -F test/minmax4.test c1fa9505fd135007fdb1fb699334fb3d4ea7952e +F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef F test/misc1.test 55cb2bfbf4a8cd61f4be1effc30426ad41696bff F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test fe55130a43e444ee75e2156ff75dc96e964b5738 @@ -624,7 +626,7 @@ F test/misc7.test 6743b810884ef64ae14c07ad1f9f858c40c06100 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a -F test/multiplex3.test 15903c343f1eaa4b00998b7ceacfc4987e4ccfe9 +F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a @@ -645,8 +647,8 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test ab9fbb673617734869af2fc4cbdb04baa3339d1c -F test/pragma.test 7fa35e53085812dac94c2bfcbb02c2a4ad35df5e +F test/permutations.test 24f0e5141b6aea74d023ac1d688c78df845d02dc +F test/pragma.test f11c59ec935a52edb4d3d5676d456588121fcefa F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -657,7 +659,7 @@ F test/quota.test af47d25c166aa7b33ef25f21bb7f2afb29d82c77 F test/quota2.test 1b8df088e604f2df573f96e726b5e518cb0cddaa F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 -F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a +F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a @@ -666,7 +668,7 @@ F test/rollback.test a1b4784b864331eae8b2a98c189efa2a8b11ff07 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 -F test/savepoint.test e575217b07d6a6e895e66f4eda076570815e0027 +F test/savepoint.test f5acd87d0c7a5f4ad6c547b47fd18c0e1aeaf048 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 @@ -677,10 +679,10 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5 F test/securedel.test 87a2561151af1f1e349071a89fdd77059f50113c -F test/select1.test f67ca2dfc05df41c7b86eb32ca409b427a5f43b0 +F test/select1.test deba017eed9daa5af33de868676c997e7eebb931 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test 44aa6e7110592e18110b0b9cf5c024d37d23be17 +F test/select4.test 00179be44e531fe04c1c3f15df216439dff2519d F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test cc25a8650cf9a4d4f74e586c45a75f9836516b18 F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe @@ -688,11 +690,11 @@ F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 +F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/session.test c1a17c11ef7d01c24fe2b9f7871190d949a8e718 F test/shared.test 34945a516532b11182c3eb26e31247eee3c9ae48 -F test/shared2.test 8f71d4eb4d5261280de92284df74172545c852cc +F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 F test/shared3.test ebf77f023f4bdaa8f74f65822b559e86ce5c6257 F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 @@ -714,7 +716,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 -F test/stat.test 36bc951bdc2beac4224cc54396fd6a7dc65336f4 +F test/stat.test 08e8185b3fd5b010c90d7ad82b9dd4ea1cbf14b0 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796 F test/subquery2.test edcad5c118f0531c2e21bf16a09bbb105252d4cd @@ -722,7 +724,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test 265cda616f56a297406728ee1e74c9b4a93aa6dd +F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -742,15 +744,15 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef -F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab +F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2d1a5c67d.test b028a811049eb472cb2d3a43fc8ce4f6894eebda F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 1f714c14b6682c5db715e0bda347926a3456f7a9 F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac -F test/tkt-38cb5df375.test 9e9b19857dba0896a8efdaf334d405ba423492f2 +F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 -F test/tkt-3a77c9714e.test 1675c22a5be71d7fa026e5db5daeeb4dd64f7824 +F test/tkt-3a77c9714e.test 32bb28afa8c63fc76e972e996193139b63551ed9 F test/tkt-3fe897352e.test 10de1a67bd5c66b238a4c96abe55531b37bb4f00 F test/tkt-4a03edc4c8.test 2865e4edbc075b954daa82f8da7cc973033ec76e F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 @@ -759,8 +761,8 @@ F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf F test/tkt-78e04e52ea.test ab52f0c1e2de6e46c910f4cc16b086bba05952b7 -F test/tkt-7bbfb7d442.test 8e7658f77d1ccea9d88dc9e255d3ed7fb68f8bdf -F test/tkt-80ba201079.test a09684db1a0bd55b8838f606adccee456a51ddbf +F test/tkt-7bbfb7d442.test dfa5c8097a8c353ae40705d6cddeb1f99c18b81a +F test/tkt-80ba201079.test 9eb040d81c404f56838a6af93593f42790def63f F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 @@ -768,15 +770,15 @@ F test/tkt-94c04eaadb.test be5ea61cb04dfdc047d19b5c5a9e75fa3da67a7f F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-b1d3a2e531.test 610ef582413171b379652663111b1f996d9f8f78 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 -F test/tkt-b72787b1.test e6b62b2b2785c04d0d698d6a603507e384165049 +F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-c48d99d690.test bed446e3513ae10eec1b86fdd186ef750226c408 F test/tkt-cbd054fa6b.test bd9fb546f63bc0c79d1776978d059fa51c5b1c63 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 -F test/tkt-d82e3f3721.test 731359dfdcdb36fea0559cd33fec39dd0ceae8e6 +F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test 669bb076f2ac573c7398ce00f40cd0ca502043a9 -F test/tkt-f777251dc7a.test 6f24c053bc5cdb7e1e19be9a72c8887cf41d5e87 +F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 1da0ed15ec2c7749fb5ce2828cd69d07153ad9f4 F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -835,7 +837,7 @@ F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d F test/tkt3522.test 22ce2ebbcb04a6be56c0977d405c207967318fd6 -F test/tkt3527.test ee4af96183579565987e58873a7490bc04934ffb +F test/tkt3527.test 9e8f28a706c772d5a7cd1020c946fab6c74e3ae0 F test/tkt3541.test 5dc257bde9bc833ab9cc6844bf170b998dbb950a F test/tkt3554.test f599967f279077bace39220cbe76085c7b423725 F test/tkt3581.test 1966b7193f1e3f14951cce8c66907ae69454e9a3 @@ -846,7 +848,7 @@ F test/tkt3731.test 0c5f4cbffe102d43c3b2188af91a9e36348f974b F test/tkt3757.test 10cd679a88675c880533083fc79ac04324525595 F test/tkt3761.test b95ea9c98f21cf91325f18a984887e62caceab33 F test/tkt3762.test 2a9f3b03df44ec49ec0cfa8d5da6574c2a7853df -F test/tkt3773.test 430b06567ce40285dfd2c4834a2a61816403efeb +F test/tkt3773.test 7bca904d2a647a6a4a291bd86d7fd7c73855b789 F test/tkt3791.test a6624b9a80b216a26cf473607f42f3e51898c267 F test/tkt3793.test d90ffd75c52413908d15e1c44fc2ea9c80fcc449 F test/tkt3810.test 90fa0635dfa7da9680c8cd3513350a49b3a8ae12 @@ -869,8 +871,8 @@ F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 -F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 -F test/trigger1.test 38524d80ac26c232d23ecec4b037eb60fb67eedd +F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trigger1.test de42feb7cd442787d38185ae74f5a1d7afa400cb F test/trigger2.test 834187beafd1db383af0c659cfa49b0576832816 F test/trigger3.test d2c60d8be271c355d61727411e753181e877230a F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -912,19 +914,20 @@ F test/vtab9.test ea58d2b95d61955f87226381716b2d0b1d4e4f9b F test/vtabA.test c86e1990b7e1e2bb34602a06fffa4c69f2b516dc F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796 F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 -F test/vtabD.test 74167b1578e5886fe4c886d6bef2fd1406444c42 +F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 -F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d +F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 F test/wal.test edefe316b4125d7f68004ea53c5e73c398d436cc -F test/wal2.test f11883dd3cb7f647c5d2acfd7b5c6d4ba5770cc9 +F test/wal2.test 8871e7fd2c86711ff415a5817d68ea3101a15312 F test/wal3.test 6504bbf348b2d6dfade64a064f1050fd617e8706 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd +F test/wal8.test 5ab217d21f7e5e86af2933a4ffd0d8357cc2c0bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test 0ab8a430ef420a3114f7092e0f30fc9585ffa155 @@ -950,7 +953,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 814d7373413398e074f134cff5f8872e2c08bd3b F test/where8.test a6c740fd286d7883e274e17b6230a9d672a7ab1f F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test bed66dcfc69a54a99661c0c9906189cb5e58f4e2 +F test/where9.test cd4ee5e455799ddba7041e5ac539044bb24e3874 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test 13ff5ec0dba407c0e0c075980c75b3275a6774e5 @@ -1002,7 +1005,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 2845654d425164de143e82b9fdb255d81a01af56 92131195d0c24c0116992db51ed5d8316626ba57 -R f655afae9aef18c1ea6dcf2304669fd0 +P 361fb66a799f4f253e61ca94d999accde2c75b2c 99a9073b5e411ce94f38ce49608baaa15de8b850 +R 73eef84d2253d0c1c19d15becb5073fc U drh -Z 91074916592f2b739d5ac72d0cab45cc +Z 84b512c9c826efc22484fef61936c099 diff --git a/manifest.uuid b/manifest.uuid index c689c5bd17..dbe8f15c1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -361fb66a799f4f253e61ca94d999accde2c75b2c \ No newline at end of file +ff86875ca35e04cea6c3d5e1b5117a4f227a6b15 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index aa3401897b..7a4047f34c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -568,7 +568,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest); + sqlite3BtreeRollback(p->pDest, SQLITE_OK); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 359253387d..68345144a8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2041,7 +2041,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p); + sqlite3BtreeRollback(p, SQLITE_OK); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3279,6 +3279,7 @@ static int countWriteCursors(BtShared *pBt){ */ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ BtCursor *p; + if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; @@ -3302,25 +3303,20 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p){ +int sqlite3BtreeRollback(Btree *p, int tripCode){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; sqlite3BtreeEnter(p); - rc = saveAllCursors(pBt, 0, 0); -#ifndef SQLITE_OMIT_SHARED_CACHE - if( rc!=SQLITE_OK ){ - /* This is a horrible situation. An IO or malloc() error occurred whilst - ** trying to save cursor positions. If this is an automatic rollback (as - ** the result of a constraint, malloc() failure or IO error) then - ** the cache may be internally inconsistent (not contain valid trees) so - ** we cannot simply return the error to the caller. Instead, abort - ** all queries that may be using any of the cursors that failed to save. - */ - sqlite3BtreeTripAllCursors(p, rc); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + sqlite3BtreeTripAllCursors(p, tripCode); } -#endif btreeIntegrity(p); if( p->inTrans==TRANS_WRITE ){ diff --git a/src/btree.h b/src/btree.h index 637dc486e1..9832001b7f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -77,7 +77,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*); +int sqlite3BtreeRollback(Btree*,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); diff --git a/src/func.c b/src/func.c index b7d0810686..c66ad28aba 100644 --- a/src/func.c +++ b/src/func.c @@ -416,7 +416,7 @@ static void randomFunc( ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ - r = -(r ^ (((sqlite3_int64)1)<<63)); + r = -(r & LARGEST_INT64); } sqlite3_result_int64(context, r); } diff --git a/src/main.c b/src/main.c index ff67a2966b..4c0fad4428 100644 --- a/src/main.c +++ b/src/main.c @@ -849,19 +849,23 @@ int sqlite3_close(sqlite3 *db){ } /* -** Rollback all database files. +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. */ -void sqlite3RollbackAll(sqlite3 *db){ +void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); for(i=0; inDb; i++){ - if( db->aDb[i].pBt ){ - if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(db->aDb[i].pBt); + sqlite3BtreeRollback(p, tripCode); db->aDb[i].inTrans = 0; } } @@ -916,12 +920,21 @@ const char *sqlite3ErrStr(int rc){ /* SQLITE_RANGE */ "bind or column index out of range", /* SQLITE_NOTADB */ "file is encrypted or is not a database", }; - rc &= 0xff; - if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ - return aMsg[rc]; - }else{ - return "unknown error"; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rc /* Needed for malloc_usable_size on linux */ #endif #ifdef HAVE_MALLOC_USABLE_SIZE diff --git a/src/os_unix.c b/src/os_unix.c index 5fad7aa2ce..48c130935e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -211,7 +211,7 @@ struct unixFile { unixInodeInfo *pInode; /* Info about locks on this inode */ int h; /* The file descriptor */ unsigned char eFileLock; /* The type of lock held on this fd */ - unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ int lastErrno; /* The unix errno from last I/O error */ void *lockingContext; /* Locking style specific state */ UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ @@ -262,6 +262,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_CHOWN 0x100 /* File ownership was changed */ /* ** Include code that is common to all os_*.c files @@ -418,6 +419,12 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + + { "umask", (sqlite3_syscall_ptr)umask, 0 }, +#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -504,11 +511,36 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ } /* -** Retry open() calls that fail due to EINTR +** Invoke open(). Do so multiple times, until it either succeeds or +** files for some reason other than EINTR. +** +** If the file creation mode "m" is 0 then set it to the default for +** SQLite. The default is SQLITE_DEFAULT_FILE_PERMISSIONS (normally +** 0644) as modified by the system umask. If m is not 0, then +** make the file creation mode be exactly m ignoring the umask. +** +** The m parameter will be non-zero only when creating -wal, -journal, +** and -shm files. We want those files to have *exactly* the same +** permissions as their original database, unadulterated by the umask. +** In that way, if a database file is -rw-rw-rw or -rw-rw-r-, and a +** transaction crashes and leaves behind hot journals, then any +** process that is able to write to the database will also be able to +** recover the hot journals. */ -static int robust_open(const char *z, int f, int m){ +static int robust_open(const char *z, int f, mode_t m){ int rc; - do{ rc = osOpen(z,f,m); }while( rc<0 && errno==EINTR ); + mode_t m2; + mode_t origM = 0; + if( m==0 ){ + m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; + }else{ + m2 = m; + origM = osUmask(0); + } + do{ rc = osOpen(z,f,m2); }while( rc<0 && errno==EINTR ); + if( m ){ + osUmask(origM); + } return rc; } @@ -3856,8 +3888,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it - ** with the same permissions. The actual permissions the file is created - ** with are subject to the current umask setting. + ** with the same permissions. */ if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ rc = SQLITE_IOERR_FSTAT; @@ -3901,10 +3932,19 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; - } + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. If this process is + ** not root, the following fchown() will fail, but we don't care. The + ** if(){..} and the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings. + */ + if( osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid)==0 ){ + pDbFd->ctrlFlags |= UNIXFILE_CHOWN; } /* Check to see if another process is holding the dead-man switch. @@ -4879,12 +4919,10 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** If the file being opened is a temporary file, it is always created with -** the octal permissions 0600 (read/writable by owner only). If the file -** is a database or master journal file, it is created with the permissions -** mask SQLITE_DEFAULT_FILE_PERMISSIONS. -** -** Finally, if the file being opened is a WAL or regular journal file, then +** In most cases cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then ** this function queries the file-system for the permissions on the ** corresponding database file and sets *pMode to this value. Whenever ** possible, WAL and journal files are created using the same permissions @@ -4898,10 +4936,14 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ int flags, /* Flags passed as 4th argument to xOpen() */ - mode_t *pMode /* OUT: Permissions to open file with */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ ){ int rc = SQLITE_OK; /* Return Code */ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + *pMode = 0; + *pUid = 0; + *pGid = 0; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ @@ -4935,6 +4977,8 @@ static int findCreateFileMode( if( 0==osStat(zDb, &sStat) ){ *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; }else{ rc = SQLITE_IOERR_FSTAT; } @@ -5081,7 +5125,9 @@ static int unixOpen( if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ - rc = findCreateFileMode(zName, flags, &openMode); + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ assert( !p->pUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); @@ -5102,6 +5148,17 @@ static int unixOpen( rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); goto open_finished; } + + /* If this process is running as root and if creating a new rollback + ** journal or WAL file, set the ownership of the journal or WAL to be + ** the same as the original database. If we are not running as root, + ** then the fchown() call will fail, but that's ok. The "if(){}" and + ** the setting of the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings from gcc. + */ + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + if( osFchown(fd, uid, gid)==0 ){ p->ctrlFlags |= UNIXFILE_CHOWN; } + } } assert( fd>=0 ); if( pOutFlags ){ @@ -5828,17 +5885,17 @@ static int proxyCreateUnixFile( } } if( fd<0 ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; if( fd<0 && errno==ENOENT && islockfile ){ if( proxyCreateLockPath(path) == SQLITE_OK ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); } } } if( fd<0 ){ openFlags = O_RDONLY; - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; } if( fd<0 ){ @@ -5962,8 +6019,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ goto end_breaklock; } /* write it out to the temporary break file */ - fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); if( fd<0 ){ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); goto end_breaklock; @@ -6240,8 +6296,7 @@ static int proxyTakeConch(unixFile *pFile){ robust_close(pFile, pFile->h, __LINE__); } pFile->h = -1; - fd = robust_open(pCtx->dbPath, pFile->openFlags, - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ pFile->h = fd; @@ -6810,7 +6865,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==20 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/pager.c b/src/pager.c index be360448f0..66252aa0f7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6871,6 +6871,20 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. +*/ +int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState==PAGER_READER ); + return sqlite3WalFramesize(pPager->pWal); +} +#endif + #ifdef SQLITE_HAS_CODEC /* ** This function is called by the wal module when writing page content diff --git a/src/pager.h b/src/pager.h index e5088bdabc..eca8a2f077 100644 --- a/src/pager.h +++ b/src/pager.h @@ -143,6 +143,9 @@ int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager); +#ifdef SQLITE_ENABLE_ZIPVFS + int sqlite3PagerWalFramesize(Pager *pPager); +#endif /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/pragma.c b/src/pragma.c index 187f9afe72..2db0b61508 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -312,9 +312,12 @@ void sqlite3Pragma( const char *zDb = 0; /* The database name */ Token *pId; /* Pointer to token */ int iDb; /* Database index for */ - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; @@ -345,6 +348,34 @@ void sqlite3Pragma( if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + if( aFcntl[0] ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); + sqlite3_free(aFcntl[0]); + } + }else if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + }else + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* @@ -632,7 +663,7 @@ void sqlite3Pragma( ** creates the database file. It is important that it is created ** as an auto-vacuum capable db. */ - int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ /* When setting the auto_vacuum mode to either "full" or ** "incremental", write the value of meta[6] in the database @@ -750,7 +781,6 @@ void sqlite3Pragma( }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ - int rc; int res; rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ diff --git a/src/select.c b/src/select.c index bbdc4e3c55..3efe014d8b 100644 --- a/src/select.c +++ b/src/select.c @@ -1258,7 +1258,7 @@ static int selectColumnsFromExprList( char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; + *pnCol = nCol = pEList ? pEList->nExpr : 0; aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); if( aCol==0 ) return SQLITE_NOMEM; for(i=0, pCol=aCol; i +**
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it @@ -719,6 +721,7 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +**
  • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified ** by the user. The fourth argument to [sqlite3_file_control()] should @@ -727,11 +730,13 @@ struct sqlite3_io_methods { ** chunks (say 1MB at a time), may reduce file-system fragmentation and ** improve performance on some systems. ** +**
  • [[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** +**
  • [[SQLITE_FCNTL_SYNC_OMITTED]] ** ^(The [SQLITE_FCNTL_SYNC_OMITTED] opcode is generated internally by ** SQLite and sent to all VFSes in place of a call to the xSync method ** when the database connection has [PRAGMA synchronous] set to OFF.)^ @@ -742,6 +747,7 @@ struct sqlite3_io_methods { ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. ** +**
  • [[SQLITE_FCNTL_WIN32_AV_RETRY]] ** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic ** retry counts and intervals for certain disk I/O operations for the ** windows [VFS] in order to provide robustness in the presence of @@ -758,6 +764,7 @@ struct sqlite3_io_methods { ** into the array entry, allowing the current retry settings to be ** interrogated. The zDbName parameter is ignored. ** +**
  • [[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the ** persistent [WAL | Write AHead Log] setting. By default, the auxiliary ** write ahead log and shared memory files used for transaction control @@ -772,6 +779,7 @@ struct sqlite3_io_methods { ** WAL mode. If the integer is -1, then it is overwritten with the current ** WAL persistence setting. ** +**
  • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] ** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the ** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting ** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the @@ -781,11 +789,13 @@ struct sqlite3_io_methods { ** mode. If the integer is -1, then it is overwritten with the current ** zero-damage mode setting. ** +**
  • [[SQLITE_FCNTL_OVERWRITE]] ** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening ** a write transaction to indicate that, unless it is rolled back for some ** reason, the entire database file will be overwritten by the current ** transaction. This is used by VACUUM operations. ** +**
  • [[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of ** all [VFSes] in the VFS stack. The names are of all VFS shims and the ** final bottom-level VFS are written into memory obtained from @@ -796,6 +806,30 @@ struct sqlite3_io_methods { ** do anything. Callers should initialize the char* variable to a NULL ** pointer in case this file-control is not implemented. This file-control ** is intended for diagnostic use only. +** +**
  • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement. ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -810,6 +844,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_OVERWRITE 11 #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 /* ** CAPI3REF: Mutex Handle @@ -6584,11 +6619,12 @@ int sqlite3_unlock_notify( /* ** CAPI3REF: String Comparison ** -** ^The [sqlite3_strnicmp()] API allows applications and extensions to -** compare the contents of two buffers containing UTF-8 strings in a -** case-independent fashion, using the same definition of case independence -** that SQLite uses internally when comparing identifiers. +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ +int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e8b2fbde43..9e15ae6b0b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2562,7 +2562,7 @@ int sqlite3CantopenError(int); /* ** Internal function prototypes */ -int sqlite3StrICmp(const char *, const char *); +#define sqlite3StrICmp sqlite3_stricmp int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -2809,7 +2809,7 @@ Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); void sqlite3PrngResetState(void); -void sqlite3RollbackAll(sqlite3*); +void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index cf59257175..60d56ee1ea 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -10,43 +10,58 @@ ** ************************************************************************* ** -** Code for demonstartion virtual table that generates variations +** Code for a demonstration virtual table that generates variations ** on an input word at increasing edit distances from the original. ** ** A fuzzer virtual table is created like this: ** -** CREATE VIRTUAL TABLE temp.f USING fuzzer; +** CREATE VIRTUAL TABLE f USING fuzzer(); ** -** The name of the new virtual table in the example above is "f". -** Note that all fuzzer virtual tables must be TEMP tables. The -** "temp." prefix in front of the table name is required when the -** table is being created. The "temp." prefix can be omitted when -** using the table as long as the name is unambiguous. +** When it is created, the new fuzzer table must be supplied with the +** name of a "fuzzer data table", which must reside in the same database +** file as the new fuzzer table. The fuzzer data table contains the various +** transformations and their costs that the fuzzer logic uses to generate +** variations. ** -** Before being used, the fuzzer needs to be programmed by giving it -** character transformations and a cost associated with each transformation. -** Examples: +** The fuzzer data table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. ** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('','a',100); +** Each row in the fuzzer data table represents a single character +** transformation. The left most column of the row (column 0) contains an +** integer value - the identifier of the ruleset to which the transformation +** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the +** row (column 0) contains the input character or characters. The third +** column contains the output character or characters. And the fourth column +** contains the integer cost of making the transformation. For example: ** -** The above statement says that the cost of inserting a letter 'a' is -** 100. (All costs are integers. We recommend that costs be scaled so -** that the average cost is around 100.) +** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); ** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('b','',87); -** -** The above statement says that the cost of deleting a single letter -** 'b' is 87. -** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('o','oe',38); -** INSERT INTO f(cFrom,cTo,Cost) VALUES('oe','o',40); -** -** This third example says that the cost of transforming the single -** letter "o" into the two-letter sequence "oe" is 38 and that the +** The first row inserted into the fuzzer data table by the SQL script +** above indicates that the cost of inserting a letter 'a' is 100. (All +** costs are integers. We recommend that costs be scaled so that the +** average cost is around 100.) The second INSERT statement creates a rule +** saying that the cost of deleting a single letter 'b' is 87. The third +** and fourth INSERT statements mean that the cost of transforming a +** single letter "o" into the two-letter sequence "oe" is 38 and that the ** cost of transforming "oe" back into "o" is 40. ** -** After all the transformation costs have been set, the fuzzer table -** can be queried as follows: +** The contents of the fuzzer data table are loaded into main memory when +** a fuzzer table is first created, and may be internally reloaded by the +** system at any subsequent time. Therefore, the fuzzer data table should be +** populated before the fuzzer table is created and not modified thereafter. +** If you do need to modify the contents of the fuzzer data table, it is +** recommended that the associated fuzzer table be dropped, the fuzzer data +** table edited, and the fuzzer table recreated within a single transaction. +** Alternatively, the fuzzer data table can be edited then the database +** connection can be closed and reopened. +** +** Once it has been created, the fuzzer table can be queried as follows: ** ** SELECT word, distance FROM f ** WHERE word MATCH 'abcdefg' @@ -61,6 +76,9 @@ ** the one that is returned. In the example, the search is limited to ** strings with a total distance of less than 200. ** +** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a fuzzer table will throw an error. +** ** It is important to put some kind of a limit on the fuzzer output. This ** can be either in the form of a LIMIT clause at the end of the query, ** or better, a "distance #include @@ -112,10 +165,25 @@ typedef struct fuzzer_seen fuzzer_seen; typedef struct fuzzer_stem fuzzer_stem; /* -** Type of the "cost" of an edit operation. Might be changed to -** "float" or "double" or "sqlite3_int64" in the future. +** Various types. +** +** fuzzer_cost is the "cost" of an edit operation. +** +** fuzzer_len is the length of a matching string. +** +** fuzzer_ruleid is an ruleset identifier. */ typedef int fuzzer_cost; +typedef signed char fuzzer_len; +typedef int fuzzer_ruleid; + +/* +** Limits +*/ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ +#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ /* @@ -123,11 +191,12 @@ typedef int fuzzer_cost; ** All rules are kept on a linked list sorted by rCost. */ struct fuzzer_rule { - fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ - fuzzer_cost rCost; /* Cost of this transformation */ - int nFrom, nTo; /* Length of the zFrom and zTo strings */ - char *zFrom; /* Transform from */ - char zTo[4]; /* Transform to (extra space appended) */ + fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from */ + fuzzer_cost rCost; /* Cost of this transformation */ + fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ + char zTo[4]; /* Transform to (extra space appended) */ }; /* @@ -143,13 +212,13 @@ struct fuzzer_rule { */ struct fuzzer_stem { char *zBasis; /* Word being fuzzed */ - int nBasis; /* Length of the zBasis string */ const fuzzer_rule *pRule; /* Current rule to apply */ - int n; /* Apply pRule at this character offset */ - fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ - fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ fuzzer_stem *pNext; /* Next stem in rCost order */ fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ + fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ + fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ + fuzzer_len nBasis; /* Length of the zBasis string */ + fuzzer_len n; /* Apply pRule at this character offset */ }; /* @@ -159,7 +228,6 @@ struct fuzzer_vtab { sqlite3_vtab base; /* Base class - must be first */ char *zClassName; /* Name of this class. Default: "fuzzer" */ fuzzer_rule *pRule; /* All active rules in this fuzzer */ - fuzzer_rule *pNewRule; /* New rules to add when last cursor expires */ int nCursor; /* Number of active cursors */ }; @@ -179,54 +247,11 @@ struct fuzzer_cursor { char *zBuf; /* Temporary use buffer */ int nBuf; /* Bytes allocated for zBuf */ int nStem; /* Number of stems allocated */ + int iRuleset; /* Only process rules from this ruleset */ fuzzer_rule nullRule; /* Null rule used first */ fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ }; -/* Methods for the fuzzer module */ -static int fuzzerConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - fuzzer_vtab *pNew; - int n; - if( strcmp(argv[1],"temp")!=0 ){ - *pzErr = sqlite3_mprintf("%s virtual tables must be TEMP", argv[0]); - return SQLITE_ERROR; - } - n = strlen(argv[0]) + 1; - pNew = sqlite3_malloc( sizeof(*pNew) + n ); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->zClassName = (char*)&pNew[1]; - memcpy(pNew->zClassName, argv[0], n); - sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,cFrom,cTo,cost)"); - memset(pNew, 0, sizeof(*pNew)); - *ppVtab = &pNew->base; - return SQLITE_OK; -} -/* Note that for this virtual table, the xCreate and xConnect -** methods are identical. */ - -static int fuzzerDisconnect(sqlite3_vtab *pVtab){ - fuzzer_vtab *p = (fuzzer_vtab*)pVtab; - assert( p->nCursor==0 ); - do{ - while( p->pRule ){ - fuzzer_rule *pRule = p->pRule; - p->pRule = pRule->pNext; - sqlite3_free(pRule); - } - p->pRule = p->pNewRule; - p->pNewRule = 0; - }while( p->pRule ); - sqlite3_free(p); - return SQLITE_OK; -} -/* The xDisconnect and xDestroy methods are also the same */ - /* ** The two input rule lists are both sorted in order of increasing ** cost. Merge them together into a single list, sorted by cost, and @@ -256,6 +281,270 @@ static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ return head.pNext; } +/* +** Statement pStmt currently points to a row in the fuzzer data table. This +** function allocates and populates a fuzzer_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int fuzzerLoadOneRule( + fuzzer_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + fuzzer_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + int nCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + fuzzer_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = strlen(zFrom); + nTo = strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + *ppRule = 0; + return SQLITE_OK; + } + + if( nCost<=0 || nCost>FUZZER_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, FUZZER_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, FUZZER_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ + *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", + p->zClassName, FUZZER_MX_RULEID + ); + rc = SQLITE_ERROR; + }else{ + + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = nTo; + pRule->rCost = nCost; + pRule->iRuleset = (int)iRuleset; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Load the content of the fuzzer data table into memory. +*/ +static int fuzzerLoadRules( + sqlite3 *db, /* Database handle */ + fuzzer_vtab *p, /* Virtual fuzzer table to configure */ + const char *zDb, /* Database containing rules data */ + const char *zData, /* Table containing rules data */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + fuzzer_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, zData, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + fuzzer_rule *pRule = 0; + rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + fuzzer_rule *pX; + fuzzer_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *fuzzerDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInnCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** xConnect/xCreate method for the fuzzer module. Arguments are: +** +** argv[0] -> module name ("fuzzer") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> fuzzer rule table name +*/ +static int fuzzerConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + fuzzer_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule + ); + rc = SQLITE_ERROR; + }else{ + int nModule; /* Length of zModule, in bytes */ + + nModule = strlen(zModule); + pNew = sqlite3_malloc( sizeof(*pNew) + nModule + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zTab; /* Dequoted name of fuzzer data table */ + + memset(pNew, 0, sizeof(*pNew)); + pNew->zClassName = (char*)&pNew[1]; + memcpy(pNew->zClassName, zModule, nModule+1); + + zTab = fuzzerDequote(argv[3]); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); + sqlite3_free(zTab); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); + } + if( rc!=SQLITE_OK ){ + fuzzerDisconnect((sqlite3_vtab *)pNew); + pNew = 0; + } + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return rc; +} /* ** Open a new fuzzer cursor. @@ -268,25 +557,6 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; *ppCursor = &pCur->base; - if( p->nCursor==0 && p->pNewRule ){ - unsigned int i; - fuzzer_rule *pX; - fuzzer_rule *a[15]; - for(i=0; ipNewRule)!=0 ){ - p->pNewRule = pX->pNext; - pX->pNext = 0; - for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); - } p->nCursor++; return SQLITE_OK; } @@ -343,8 +613,8 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; - char *z; + int n; /* Size of output term without nul-term */ + char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; if( (*pnBuf)nTo], &pStem->zBasis[n+pRule->nFrom], pStem->nBasis-n-pRule->nFrom+1); } + + assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); return SQLITE_OK; } @@ -424,12 +696,31 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } h = fuzzerHash(pCur->zBuf); pLookup = pCur->apHash[h]; - while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ pLookup = pLookup->pHash; } return pLookup!=0; } +/* +** If argument pRule is NULL, this function returns false. +** +** Otherwise, it returns true if rule pRule should be skipped. A rule +** should be skipped if it does not belong to rule-set iRuleset, or if +** applying it to stem pStem would create a string longer than +** FUZZER_MX_OUTPUT_LENGTH bytes. +*/ +static int fuzzerSkipRule( + const fuzzer_rule *pRule, /* Determine whether or not to skip this */ + fuzzer_stem *pStem, /* Stem rule may be applied to */ + int iRuleset /* Rule-set used by the current query */ +){ + return pRule && ( + (pRule->iRuleset!=iRuleset) + || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH + ); +} + /* ** Advance a fuzzer_stem to its next value. Return 0 if there are ** no more values that can be generated by this fuzzer_stem. Return @@ -438,6 +729,7 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; while( (pRule = pStem->pRule)!=0 ){ + assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); while( pStem->n < pStem->nBasis - pRule->nFrom ){ pStem->n++; if( pRule->nFrom==0 @@ -453,8 +745,11 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } } pStem->n = -1; - pStem->pRule = pRule->pNext; - if( pStem->pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + do{ + pRule = pRule->pNext; + }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); + pStem->pRule = pRule; + if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } return 0; } @@ -572,6 +867,7 @@ static fuzzer_stem *fuzzerNewStem( fuzzer_cost rBaseCost ){ fuzzer_stem *pNew; + fuzzer_rule *pRule; unsigned int h; pNew = sqlite3_malloc( sizeof(*pNew) + strlen(zWord) + 1 ); @@ -580,7 +876,11 @@ static fuzzer_stem *fuzzerNewStem( pNew->zBasis = (char*)&pNew[1]; pNew->nBasis = strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); - pNew->pRule = pCur->pVtab->pRule; + pRule = pCur->pVtab->pRule; + while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ + pRule = pRule->pNext; + } + pNew->pRule = pRule; pNew->n = -1; pNew->rBaseCost = pNew->rCostX = rBaseCost; h = fuzzerHash(pNew->zBasis); @@ -627,7 +927,10 @@ static int fuzzerNext(sqlite3_vtab_cursor *cur){ ** stem list is the next lowest cost word. */ while( (pStem = pCur->pStem)!=0 ){ - if( fuzzerAdvance(pCur, pStem) ){ + int res = fuzzerAdvance(pCur, pStem); + if( res<0 ){ + return SQLITE_NOMEM; + }else if( res>0 ){ pCur->pStem = 0; pStem = fuzzerInsert(pCur, pStem); if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ @@ -665,30 +968,44 @@ static int fuzzerFilter( int argc, sqlite3_value **argv ){ fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; - const char *zWord = 0; + const char *zWord = ""; fuzzer_stem *pStem; + int idx; fuzzerClearCursor(pCur, 1); pCur->rLimit = 2147483647; - if( idxNum==1 ){ + idx = 0; + if( idxNum & 1 ){ zWord = (const char*)sqlite3_value_text(argv[0]); - }else if( idxNum==2 ){ - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[0]); - }else if( idxNum==3 ){ - zWord = (const char*)sqlite3_value_text(argv[0]); - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[1]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; } - if( zWord==0 ) zWord = ""; - pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); - if( pStem==0 ) return SQLITE_NOMEM; pCur->nullRule.pNext = pCur->pVtab->pRule; pCur->nullRule.rCost = 0; pCur->nullRule.nFrom = 0; pCur->nullRule.nTo = 0; pCur->nullRule.zFrom = ""; - pStem->pRule = &pCur->nullRule; - pStem->n = pStem->nBasis; pCur->iRowid = 1; + assert( pCur->pStem==0 ); + + /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this + ** query will return zero rows. */ + if( strlen(zWord)pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; + }else{ + pCur->rLimit = 0; + } + return SQLITE_OK; } @@ -735,22 +1052,29 @@ static int fuzzerEof(sqlite3_vtab_cursor *cur){ /* ** Search for terms of these forms: ** -** word MATCH $str -** distance < $value -** distance <= $value +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) ruleid == $ruleid ** ** The distance< and distance<= are both treated as distance<=. -** The query plan number is as follows: +** The query plan number is a bit vector: ** -** 0: None of the terms above are found -** 1: There is a "word MATCH" term with $str in filter.argv[0]. -** 2: There is a "distance<" term with $value in filter.argv[0]. -** 3: Both "word MATCH" and "distance<" with $str in argv[0] and -** $value in argv[1]. +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. */ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iPlan = 0; int iDistTerm = -1; + int iRulesetTerm = -1; int i; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; @@ -772,11 +1096,23 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 2; iDistTerm = i; } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iRulesetTerm = i; + } } - if( iPlan==2 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1; - }else if( iPlan==3 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 2; + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; } pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 @@ -790,70 +1126,6 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_OK; } -/* -** Disallow all attempts to DELETE or UPDATE. Only INSERTs are allowed. -** -** On an insert, the cFrom, cTo, and cost columns are used to construct -** a new rule. All other columns are ignored. The rule is ignored -** if cFrom and cTo are identical. A NULL value for cFrom or cTo is -** interpreted as an empty string. The cost must be positive. -*/ -static int fuzzerUpdate( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - fuzzer_vtab *p = (fuzzer_vtab*)pVTab; - fuzzer_rule *pRule; - const char *zFrom; - int nFrom; - const char *zTo; - int nTo; - fuzzer_cost rCost; - if( argc!=7 ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot delete from a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot update a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - zFrom = (char*)sqlite3_value_text(argv[4]); - if( zFrom==0 ) zFrom = ""; - zTo = (char*)sqlite3_value_text(argv[5]); - if( zTo==0 ) zTo = ""; - if( strcmp(zFrom,zTo)==0 ){ - /* Silently ignore null transformations */ - return SQLITE_OK; - } - rCost = sqlite3_value_int(argv[6]); - if( rCost<=0 ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cost must be positive"); - return SQLITE_CONSTRAINT; - } - nFrom = strlen(zFrom); - nTo = strlen(zTo); - pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); - if( pRule==0 ){ - return SQLITE_NOMEM; - } - pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; - memcpy(pRule->zFrom, zFrom, nFrom+1); - memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; - pRule->rCost = rCost; - pRule->pNext = p->pNewRule; - p->pNewRule = pRule; - return SQLITE_OK; -} - /* ** A virtual table module that provides read-only access to a ** Tcl global variable namespace. @@ -872,7 +1144,7 @@ static sqlite3_module fuzzerModule = { fuzzerEof, /* xEof - check for end of scan */ fuzzerColumn, /* xColumn - read data */ fuzzerRowid, /* xRowid - read data */ - fuzzerUpdate, /* xUpdate - INSERT */ + 0, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ @@ -916,7 +1188,7 @@ static int register_fuzzer_module( Tcl_WrongNumArgs(interp, 1, objv, "DB"); return TCL_ERROR; } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + getDbPointer(interp, Tcl_GetString(objv[1]), &db); fuzzer_register(db); return TCL_OK; } diff --git a/src/test_multiplex.c b/src/test_multiplex.c index af41c0b61c..e2b6720652 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -81,8 +81,12 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ +/* Maximum chunk number */ +#define MX_CHUNK_NUMBER 299 + /* First chunk for rollback journal files */ #define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400 +#define SQLITE_MULTIPLEX_WAL_8_3_OFFSET 700 /************************ Shim Definitions ******************************/ @@ -251,17 +255,22 @@ static void multiplexFilename( ){ int n = nBase; memcpy(zOut, zBase, n+1); - if( iChunk!=0 && iChunk!=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){ + if( iChunk!=0 && iChunk<=MX_CHUNK_NUMBER ){ #ifdef SQLITE_ENABLE_8_3_NAMES int i; for(i=n-1; i>0 && i>=n-4 && zOut[i]!='.'; i--){} if( i>=n-4 ) n = i+1; if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ /* The extensions on overflow files for main databases are 001, 002, - ** 003 and so forth. To avoid name collisions, add 400 to the - ** extensions of journal files so that they are 401, 402, 403, .... - */ + ** 003 and so forth. To avoid name collisions, add 400 to the + ** extensions of journal files so that they are 401, 402, 403, .... + */ iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET; + }else if( flags & SQLITE_OPEN_WAL ){ + /* To avoid name collisions, add 700 to the + ** extensions of WAL files so that they are 701, 702, 703, .... + */ + iChunk += SQLITE_MULTIPLEX_WAL_8_3_OFFSET; } #endif sqlite3_snprintf(4,&zOut[n],"%03d",iChunk); @@ -648,6 +657,17 @@ static int multiplexDelete( multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, --iChunk, z); rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); } + if( rc==SQLITE_OK ){ + iChunk = 0; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } + } } sqlite3_free(z); } diff --git a/src/test_vfs.c b/src/test_vfs.c index a79407b57f..bab67798cc 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -480,6 +480,27 @@ static int tvfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int tvfsFileControl(sqlite3_file *pFile, int op, void *pArg){ TestvfsFd *p = tvfsGetFd(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **argv = (char**)pArg; + if( sqlite3_stricmp(argv[1],"error")==0 ){ + int rc = SQLITE_ERROR; + if( argv[2] ){ + const char *z = argv[2]; + int x = atoi(z); + if( x ){ + rc = x; + while( sqlite3Isdigit(z[0]) ){ z++; } + while( sqlite3Isspace(z[0]) ){ z++; } + } + if( z[0] ) argv[0] = sqlite3_mprintf("%s", z); + } + return rc; + } + if( sqlite3_stricmp(argv[1], "filename")==0 ){ + argv[0] = sqlite3_mprintf("%s", p->zFilename); + return SQLITE_OK; + } + } return sqlite3OsFileControl(p->pReal, op, pArg); } diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index 62577207b9..3a0e2cf03e 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -476,6 +476,12 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + case SQLITE_FCNTL_PRAGMA: { + const char *const* a = (const char*const*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); + zOp = zBuf; + break; + } default: { sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); zOp = zBuf; @@ -490,6 +496,10 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", pInfo->zVfsName, *(char**)pArg); } + if( op==SQLITE_FCNTL_PRAGMA && rc==SQLITE_OK && *(char**)pArg ){ + vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", + pInfo->zVfsName, p->zFName, zOp, *(char**)pArg); + } return rc; } diff --git a/src/util.c b/src/util.c index fd3c858ab7..325c75aae1 100644 --- a/src/util.c +++ b/src/util.c @@ -222,7 +222,7 @@ int sqlite3Dequote(char *z){ ** definition of case independence that SQLite uses internally when ** comparing identifiers. */ -int sqlite3StrICmp(const char *zLeft, const char *zRight){ +int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; diff --git a/src/vacuum.c b/src/vacuum.c index 58a3c9d683..c03b4500b3 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -176,6 +176,18 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ } #endif + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, 2); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) ==PAGER_JOURNALMODE_WAL ){ @@ -189,20 +201,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ){ - goto end_of_vacuum; - } #ifndef SQLITE_OMIT_AUTOVACUUM sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : sqlite3BtreeGetAutoVacuum(pMain)); #endif - /* Begin a transaction */ - rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ diff --git a/src/vdbe.c b/src/vdbe.c index 8d94f8eb16..ca70ea7a79 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2694,16 +2694,12 @@ case OP_Savepoint: { if( !pSavepoint ){ sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); rc = SQLITE_ERROR; - }else if( - db->writeVdbeCnt>0 || (p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) - ){ + }else if( db->writeVdbeCnt>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are - ** active write statements. It is not possible to rollback a savepoint - ** if there are any active statements at all. + ** active write statements. */ sqlite3SetString(&p->zErrMsg, db, - "cannot %s savepoint - SQL statements in progress", - (p1==SAVEPOINT_ROLLBACK ? "rollback": "release") + "cannot release savepoint - SQL statements in progress" ); rc = SQLITE_BUSY; }else{ @@ -2728,6 +2724,9 @@ case OP_Savepoint: { rc = p->rc; }else{ iSavepoint = db->nSavepoint - iSavepoint - 1; + for(ii=0; iinDb; ii++){ + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); if( rc!=SQLITE_OK ){ @@ -2796,6 +2795,7 @@ case OP_AutoCommit: { assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->activeVdbeCnt>0 ); /* At least this one VM is active */ +#if 0 if( turnOnAC && iRollback && db->activeVdbeCnt>1 ){ /* If this instruction implements a ROLLBACK and other VMs are ** still running, and a transaction is active, return an error indicating @@ -2804,7 +2804,9 @@ case OP_AutoCommit: { sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " "SQL statements in progress"); rc = SQLITE_BUSY; - }else if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ + }else +#endif + if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ @@ -2814,7 +2816,7 @@ case OP_AutoCommit: { }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; @@ -3852,7 +3854,7 @@ case OP_NewRowid: { /* out2-prerelease */ assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ - if( v==MAX_ROWID ){ + if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ v++; /* IMP: R-29538-34987 */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 93daaac6d2..48bc7551f1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2003,32 +2003,6 @@ static void checkActiveVdbeCnt(sqlite3 *db){ #define checkActiveVdbeCnt(x) #endif -/* -** For every Btree that in database connection db which -** has been modified, "trip" or invalidate each cursor in -** that Btree might have been modified so that the cursor -** can never be used again. This happens when a rollback -*** occurs. We have to trip all the other cursors, even -** cursor from other VMs in different database connections, -** so that none of them try to use the data at which they -** were pointing and which now may have been changed due -** to the rollback. -** -** Remember that a rollback can delete tables complete and -** reorder rootpages. So it is not sufficient just to save -** the state of the cursor. We have to invalidate the cursor -** so that it is never used again. -*/ -static void invalidateCursorsOnModifiedBtrees(sqlite3 *db){ - int i; - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p && sqlite3BtreeIsInTrans(p) ){ - sqlite3BtreeTripAllCursors(p, SQLITE_ABORT); - } - } -} - /* ** If the Vdbe passed as the first argument opened a statement-transaction, ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or @@ -2193,8 +2167,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* We are forced to roll back the active transaction. Before doing ** so, abort any other statements this handle currently has active. */ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2236,13 +2209,13 @@ int sqlite3VdbeHalt(Vdbe *p){ return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ p->rc = rc; - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); }else{ db->nDeferredCons = 0; sqlite3CommitInternalChanges(db); } }else{ - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2251,8 +2224,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( p->errorAction==OE_Abort ){ eStatementOp = SAVEPOINT_ROLLBACK; }else{ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2272,8 +2244,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } diff --git a/src/wal.c b/src/wal.c index db9ce5186f..b077d27d9e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2397,7 +2397,7 @@ int sqlite3WalRead( iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); } *pInWal = 0; @@ -3068,4 +3068,16 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** If the argument is not NULL, it points to a Wal object that holds a +** read-lock. This function returns the database page-size if it is known, +** or zero if it is not (or if pWal is NULL). +*/ +int sqlite3WalFramesize(Wal *pWal){ + assert( pWal==0 || pWal->readLock>=0 ); + return (pWal ? pWal->szPage : 0); +} +#endif + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index fac762e870..a848de1527 100644 --- a/src/wal.h +++ b/src/wal.h @@ -43,6 +43,7 @@ # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalFramesize(z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -124,5 +125,12 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_ZIPVFS +/* If the WAL file is not empty, return the number of bytes of content +** stored in each frame (i.e. the db page-size when the WAL was created). +*/ +int sqlite3WalFramesize(Wal *pWal); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ diff --git a/src/where.c b/src/where.c index 78efbf9790..eda8d5fb8e 100644 --- a/src/where.c +++ b/src/where.c @@ -3103,7 +3103,9 @@ static void bestBtreeIndex( /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) + && (wsFlags & WHERE_COLUMN_IN)==0 + ){ bDist = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } diff --git a/test/cache.test b/test/cache.test index 8d801f0599..f81948ba2c 100644 --- a/test/cache.test +++ b/test/cache.test @@ -14,7 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable {!pager_pragmas} { +ifcapable !pager_pragmas||!compound { finish_test return } diff --git a/test/capi3.test b/test/capi3.test index cbba04cd84..a9aab1e615 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -894,18 +894,19 @@ do_test capi3-11.9.2 { catchsql { ROLLBACK; } -} {1 {cannot rollback transaction - SQL statements in progress}} +} {0 {}} do_test capi3-11.9.3 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_ERROR} do_test capi3-11.11 { sqlite3_step $STMT } {SQLITE_ROW} do_test capi3-11.12 { sqlite3_step $STMT + sqlite3_step $STMT } {SQLITE_DONE} do_test capi3-11.13 { sqlite3_finalize $STMT @@ -914,15 +915,15 @@ do_test capi3-11.14 { execsql { SELECT a FROM t2; } -} {1 2 3} +} {1 2} do_test capi3-11.14.1 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3-11.15 { catchsql { ROLLBACK; } -} {0 {}} +} {1 {cannot rollback - no transaction is active}} do_test capi3-11.15.1 { sqlite3_get_autocommit $DB } 1 diff --git a/test/capi3c.test b/test/capi3c.test index c1d5510bd8..adef7f9189 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -849,18 +849,19 @@ do_test capi3c-11.9.2 { catchsql { ROLLBACK; } -} {1 {cannot rollback transaction - SQL statements in progress}} +} {0 {}} do_test capi3c-11.9.3 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_ABORT} do_test capi3c-11.11 { sqlite3_step $STMT } {SQLITE_ROW} do_test capi3c-11.12 { sqlite3_step $STMT + sqlite3_step $STMT } {SQLITE_DONE} do_test capi3c-11.13 { sqlite3_finalize $STMT @@ -869,15 +870,15 @@ do_test capi3c-11.14 { execsql { SELECT a FROM t2; } -} {1 2 3} +} {1 2} do_test capi3c-11.14.1 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3c-11.15 { catchsql { ROLLBACK; } -} {0 {}} +} {1 {cannot rollback - no transaction is active}} do_test capi3c-11.15.1 { sqlite3_get_autocommit $DB } 1 diff --git a/test/dbstatus.test b/test/dbstatus.test index 5fc8be0426..9793df3f4e 100644 --- a/test/dbstatus.test +++ b/test/dbstatus.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Memory statistics must be enabled for this test. db close sqlite3_shutdown diff --git a/test/distinct.test b/test/distinct.test index 3fb9c8478a..3bc8337942 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix distinct diff --git a/test/e_delete.test b/test/e_delete.test index 91459810f9..31bb3249ba 100644 --- a/test/e_delete.test +++ b/test/e_delete.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + proc do_delete_tests {args} { uplevel do_select_tests $args } diff --git a/test/e_expr.test b/test/e_expr.test index c770f1c410..74d0c40712 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -17,6 +17,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +ifcapable !compound { + finish_test + return +} proc do_expr_test {tn expr type value} { uplevel do_execsql_test $tn [list "SELECT typeof($expr), $expr"] [ diff --git a/test/e_fkey.test b/test/e_fkey.test index ae789d5072..5b27e03b5a 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -2325,7 +2325,7 @@ do_test e_fkey-51.1 { do_test e_fkey-51.2 { execsql { UPDATE parent SET x = 22; - SELECT * FROM parent UNION ALL SELECT 'xxx' UNION ALL SELECT a FROM child; + SELECT * FROM parent ; SELECT 'xxx' ; SELECT a FROM child; } } {22 21 23 xxx 22} do_test e_fkey-51.3 { @@ -2335,7 +2335,7 @@ do_test e_fkey-51.3 { INSERT INTO parent VALUES(-1); INSERT INTO child VALUES(-1); UPDATE parent SET x = 22; - SELECT * FROM parent UNION ALL SELECT 'xxx' UNION ALL SELECT a FROM child; + SELECT * FROM parent ; SELECT 'xxx' ; SELECT a FROM child; } } {22 23 21 xxx 23} diff --git a/test/e_insert.test b/test/e_insert.test index e96b1a6f6e..a10d5283de 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Organization of tests: # # e_insert-0.*: Test the syntax diagram. diff --git a/test/e_select.test b/test/e_select.test index 103472cd92..e5949af67b 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_execsql_test e_select-1.0 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES('a', 'one'); diff --git a/test/e_update.test b/test/e_update.test index 6c0e5f8aa6..230c97fdfa 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -381,11 +381,9 @@ do_execsql_test e_update-2.2.X { # attached). # do_execsql_test e_update-2.3.0 { - SELECT 'main', tbl_name FROM main.sqlite_master WHERE type = 'table' - UNION ALL - SELECT 'temp', tbl_name FROM sqlite_temp_master WHERE type = 'table' - UNION ALL - SELECT 'aux', tbl_name FROM aux.sqlite_master WHERE type = 'table' + SELECT 'main', tbl_name FROM main.sqlite_master WHERE type = 'table'; + SELECT 'temp', tbl_name FROM sqlite_temp_master WHERE type = 'table'; + SELECT 'aux', tbl_name FROM aux.sqlite_master WHERE type = 'table'; } [list {*}{ main t1 main t2 diff --git a/test/e_vacuum.test b/test/e_vacuum.test index aa7870fd10..bad12d3c4d 100644 --- a/test/e_vacuum.test +++ b/test/e_vacuum.test @@ -122,7 +122,7 @@ foreach {tn avmode sz} { # e_vacuum-1.2.4 - Verify that t1 and its indexes are now much # less fragmented. # -ifcapable vtab { +ifcapable vtab&&compound { create_db register_dbstat_vtab db do_execsql_test e_vacuum-1.2.1 { diff --git a/test/eqp.test b/test/eqp.test index 91a18d0026..0e663f0a38 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -13,6 +13,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix eqp #------------------------------------------------------------------------- diff --git a/test/fts4langid.test b/test/fts4langid.test new file mode 100644 index 0000000000..08f1a21c17 --- /dev/null +++ b/test/fts4langid.test @@ -0,0 +1,385 @@ +# 2012 March 01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the languageid=xxx FTS4 option. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts4content + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set ::testprefix fts4langid + +#--------------------------------------------------------------------------- +# Test plan: +# +# 1.* - Warm-body tests created for specific purposes during development. +# Passing these doesn't really prove much. +# +# 2.1.* - Test that FTS queries only ever return rows associated with +# the requested language. +# +# 2.2.* - Same as 2.1.*, after an 'optimize' command. +# +# 2.3.* - Same as 2.1.*, after a 'rebuild' command. +# +# 3.* - Tests with content= tables. Both where there is a real +# underlying content table and where there is not. +# +# 4.* - Test that if one is provided, the tokenizer xLanguage method +# is called to configure the tokenizer before tokenizing query +# or document text. +# +# 5.* - Test the fts4aux table when the associated FTS4 table contains +# multiple languages. +# + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); +} + +do_execsql_test 1.2 { + SELECT sql FROM sqlite_master WHERE name = 't1_content'; +} {{CREATE TABLE 't1_content'(docid INTEGER PRIMARY KEY, 'c0a', 'c1b', langid)}} + +do_execsql_test 1.3 {SELECT docid FROM t1} {} +do_execsql_test 1.4 {SELECT lang_id FROM t1} {} + +do_execsql_test 1.5 {INSERT INTO t1(a, b) VALUES('aaa', 'bbb')} +do_execsql_test 1.6 {SELECT lang_id FROM t1 } {0} + +do_execsql_test 1.7 {INSERT INTO t1(a, b, lang_id) VALUES('aaa', 'bbb', 4)} +do_execsql_test 1.8 {SELECT lang_id FROM t1 } {0 4} + +do_execsql_test 1.9 {INSERT INTO t1(a, b, lang_id) VALUES('aaa', 'bbb', 'xyz')} +do_execsql_test 1.10 {SELECT lang_id FROM t1} {0 4 0} + +do_execsql_test 1.11 { + CREATE VIRTUAL TABLE t2 USING fts4; + INSERT INTO t2 VALUES('abc'); +} +do_execsql_test 1.12 { SELECT rowid FROM t2 WHERE content MATCH 'abc' } 1 + +do_execsql_test 1.13 { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4(languageid=lang_id); + INSERT INTO t1(content) VALUES('a b c'); + INSERT INTO t1(content, lang_id) VALUES('a b c', 1); +} + +do_execsql_test 1.14 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b'; +} {1} +do_execsql_test 1.15 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 0; +} {1} + +do_execsql_test 1.16 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 1; +} {2} + +do_catchsql_test 1.17 { + INSERT INTO t1(content, lang_id) VALUES('123', -1); +} {1 {constraint failed}} + +do_execsql_test 1.18 { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4(languageid=lang_id); + INSERT INTO t1(content, lang_id) VALUES('A', 13); + INSERT INTO t1(content, lang_id) VALUES('B', 13); + INSERT INTO t1(content, lang_id) VALUES('C', 13); + INSERT INTO t1(content, lang_id) VALUES('D', 13); + INSERT INTO t1(content, lang_id) VALUES('E', 13); + INSERT INTO t1(content, lang_id) VALUES('F', 13); + INSERT INTO t1(content, lang_id) VALUES('G', 13); + INSERT INTO t1(content, lang_id) VALUES('H', 13); + INSERT INTO t1(content, lang_id) VALUES('I', 13); + INSERT INTO t1(content, lang_id) VALUES('J', 13); + INSERT INTO t1(content, lang_id) VALUES('K', 13); + INSERT INTO t1(content, lang_id) VALUES('L', 13); + INSERT INTO t1(content, lang_id) VALUES('M', 13); + INSERT INTO t1(content, lang_id) VALUES('N', 13); + INSERT INTO t1(content, lang_id) VALUES('O', 13); + INSERT INTO t1(content, lang_id) VALUES('P', 13); + INSERT INTO t1(content, lang_id) VALUES('Q', 13); + INSERT INTO t1(content, lang_id) VALUES('R', 13); + INSERT INTO t1(content, lang_id) VALUES('S', 13); + SELECT rowid FROM t1 WHERE t1 MATCH 'A'; +} {} + + +#------------------------------------------------------------------------- +# Test cases 2.* +# +proc build_multilingual_db_1 {db} { + $db eval { CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l) } + + set xwords [list zero one two three four five six seven eight nine ten] + set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] + + for {set i 0} {$i < 1000} {incr i} { + set iLangid [expr $i%9] + set x "" + set y "" + + set x [list] + lappend x [lindex $xwords [expr ($i / 1000) % 10]] + lappend x [lindex $xwords [expr ($i / 100) % 10]] + lappend x [lindex $xwords [expr ($i / 10) % 10]] + lappend x [lindex $xwords [expr ($i / 1) % 10]] + + set y [list] + lappend y [lindex $ywords [expr ($i / 1000) % 10]] + lappend y [lindex $ywords [expr ($i / 100) % 10]] + lappend y [lindex $ywords [expr ($i / 10) % 10]] + lappend y [lindex $ywords [expr ($i / 1) % 10]] + + $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) } + } + + $db eval { + CREATE TABLE data(x, y, l); + INSERT INTO data(rowid, x, y, l) SELECT docid, x, y, l FROM t2; + } +} + +proc rowid_list_set_langid {langid} { + set ::rowid_list_langid $langid +} +proc rowid_list {pattern} { + set langid $::rowid_list_langid + set res [list] + db eval {SELECT rowid, x, y FROM data WHERE l = $langid ORDER BY rowid ASC} { + if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} { + lappend res $rowid + } + } + return $res +} + +proc or_merge_list {list1 list2} { + set res [list] + + set i1 0 + set i2 0 + + set n1 [llength $list1] + set n2 [llength $list2] + + while {$i1 < $n1 && $i2 < $n2} { + set e1 [lindex $list1 $i1] + set e2 [lindex $list2 $i2] + + if {$e1==$e2} { + lappend res $e1 + incr i1 + incr i2 + } elseif {$e1 < $e2} { + lappend res $e1 + incr i1 + } else { + lappend res $e2 + incr i2 + } + } + + concat $res [lrange $list1 $i1 end] [lrange $list2 $i2 end] +} + +proc or_merge_lists {args} { + set res [lindex $args 0] + for {set i 1} {$i < [llength $args]} {incr i} { + set res [or_merge_list $res [lindex $args $i]] + } + set res +} + +proc and_merge_list {list1 list2} { + foreach i $list2 { set a($i) 1 } + set res [list] + foreach i $list1 { + if {[info exists a($i)]} {lappend res $i} + } + set res +} + + +proc and_merge_lists {args} { + set res [lindex $args 0] + for {set i 1} {$i < [llength $args]} {incr i} { + set res [and_merge_list $res [lindex $args $i]] + } + set res +} + +proc filter_list {list langid} { + set res [list] + foreach i $list { + if {($i % 9) == $langid} {lappend res $i} + } + set res +} + +do_test 2.0 { + reset_db + build_multilingual_db_1 db +} {} + +proc do_test_query1 {tn query res_script} { + for {set langid 0} {$langid < 10} {incr langid} { + rowid_list_set_langid $langid + set res [eval $res_script] + + set actual [ + execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} + ] + do_test $tn.$langid [list set {} $actual] $res + } +} + +# Run some queries. +do_test_query1 2.1.1 {delta} { rowid_list delta } +do_test_query1 2.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.1.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 2.1.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +# Now try the same tests as above, but after running the 'optimize' +# command on the FTS table. +# +do_execsql_test 2.2 { + INSERT INTO t2(t2) VALUES('optimize'); + SELECT count(*) FROM t2_segdir; +} {9} +do_test_query1 2.2.1 {delta} { rowid_list delta } +do_test_query1 2.2.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.2.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 2.2.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +# And rebuild. +# +do_test 2.3 { + reset_db + build_multilingual_db_1 db + execsql { INSERT INTO t2(t2) VALUES('rebuild') } +} {} +do_test_query1 2.3.1 {delta} { rowid_list delta } +do_test_query1 2.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.3.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 2.3.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +#------------------------------------------------------------------------- +# Test cases 3.* +# +do_test 3.0 { + reset_db + build_multilingual_db_1 db + execsql { + CREATE TABLE t3_data(l, x, y); + INSERT INTO t3_data(rowid, l, x, y) SELECT docid, l, x, y FROM t2; + DROP TABLE t2; + } +} {} +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(content=t3_data, languageid=l); + INSERT INTO t2(t2) VALUES('rebuild'); +} + +do_test_query1 3.1.1 {delta} { rowid_list delta } +do_test_query1 3.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.1.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.1.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +do_execsql_test 3.2.1 { + DROP TABLE t2; + CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l, content=nosuchtable); +} + +do_execsql_test 3.2.2 { + INSERT INTO t2(docid, x, y, l) SELECT rowid, x, y, l FROM t3_data; +} + +do_execsql_test 3.2.3 { + DROP TABLE t3_data; +} + +do_test_query1 3.3.1 {delta} { rowid_list delta } +do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.3.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.3.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +#------------------------------------------------------------------------- +# Test cases 4.* +# +proc build_multilingual_db_2 {db} { + $db eval { + CREATE VIRTUAL TABLE t4 USING fts4( + tokenize=testtokenizer, + languageid=lid + ); + } + for {set i 0} {$i < 50} {incr i} { + execsql { + INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i) + } + } +} + +do_test 4.1.0 { + reset_db + set ptr [fts3_test_tokenizer] + execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } + build_multilingual_db_2 db +} {} +do_execsql_test 4.1.1 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick'; +} {0} +do_execsql_test 4.1.2 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; +} {} +do_execsql_test 4.1.3 { + SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; +} {1} +for {set i 0} {$i < 50} {incr i} { + do_execsql_test 4.1.4.$i { + SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; + } [expr 0==($i%2)] +} +do_catchsql_test 4.1.5 { + INSERT INTO t4(content, lid) VALUES('hello world', 101) +} {1 {SQL logic error or missing database}} + +finish_test diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 6c23211859..dc8b445ee9 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -22,100 +22,233 @@ ifcapable !vtab { return } +set ::testprefix fuzzer1 + +# Test of test code. Only here to make the coverage metric better. +do_test 0.1 { + list [catch { register_fuzzer_module a b c } msg] $msg +} {1 {wrong # args: should be "register_fuzzer_module DB"}} + register_fuzzer_module db -do_test fuzzer1-1.0 { - catchsql {CREATE VIRTUAL TABLE fault1 USING fuzzer;} -} {1 {fuzzer virtual tables must be TEMP}} -do_test fuzzer1-1.1 { - db eval {CREATE VIRTUAL TABLE temp.f1 USING fuzzer;} -} {} -do_test fuzzer1-1.2 { - db eval { - INSERT INTO f1(cfrom, cto, cost) VALUES('e','a',1); - INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',10); - INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',100); - } + +# Check configuration errors. +# +do_catchsql_test fuzzer1-1.1 { + CREATE VIRTUAL TABLE f USING fuzzer; +} {1 {fuzzer: wrong number of CREATE VIRTUAL TABLE arguments}} + +do_catchsql_test fuzzer1-1.2 { + CREATE VIRTUAL TABLE f USING fuzzer(one, two); +} {1 {fuzzer: wrong number of CREATE VIRTUAL TABLE arguments}} + +do_catchsql_test fuzzer1-1.3 { + CREATE VIRTUAL TABLE f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: main.nosuchtable}} + +do_catchsql_test fuzzer1-1.4 { + CREATE TEMP TABLE nosuchtable(a, b, c, d); + CREATE VIRTUAL TABLE f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: main.nosuchtable}} + +do_catchsql_test fuzzer1-1.5 { + DROP TABLE temp.nosuchtable; + CREATE TABLE nosuchtable(a, b, c, d); + CREATE VIRTUAL TABLE temp.f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: temp.nosuchtable}} + +do_catchsql_test fuzzer1-1.6 { + DROP TABLE IF EXISTS f_rules; + CREATE TABLE f_rules(a, b, c); + CREATE VIRTUAL TABLE f USING fuzzer(f_rules); +} {1 {fuzzer: f_rules has 3 columns, expected 4}} + +do_catchsql_test fuzzer1-1.7 { + DROP TABLE IF EXISTS f_rules; + CREATE TABLE f_rules(a, b, c, d, e); + CREATE VIRTUAL TABLE f USING fuzzer(f_rules); +} {1 {fuzzer: f_rules has 5 columns, expected 4}} + + +do_execsql_test fuzzer1-2.1 { + CREATE TABLE f1_rules(ruleset DEFAULT 0, cfrom, cto, cost); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('e','a',1); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('a','e',10); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('e','o',100); + + CREATE VIRTUAL TABLE f1 USING fuzzer(f1_rules); } {} -do_test fuzzer1-1.3 { - db eval { +do_execsql_test fuzzer1-2.1 { SELECT word, distance FROM f1 WHERE word MATCH 'abcde' +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} + +do_execsql_test fuzzer1-2.4 { + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'b','x',1); + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'d','y',10); + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'y','z',100); + + DROP TABLE f1; + CREATE VIRTUAL TABLE f1 USING fuzzer(f1_rules); +} {} + +do_execsql_test fuzzer1-2.5 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} + +do_execsql_test fuzzer1-2.6 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=0 +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} + +do_execsql_test fuzzer1-2.7 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=1 +} { + abcde 0 axcde 1 abcye 10 + axcye 11 abcze 110 axcze 111 +} + +do_test fuzzer1-1.8 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<100 } -} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} +} {abcde 0 abcda 1 ebcde 10 ebcda 11} +do_test fuzzer1-1.9 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<=100 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100} +do_test fuzzer1-1.10 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<100 AND ruleset=0 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11} +do_test fuzzer1-1.11 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<=100 AND ruleset=0 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100} +do_test fuzzer1-1.12 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<11 AND ruleset=1 + } +} {abcde 0 axcde 1 abcye 10} +do_test fuzzer1-1.13 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<=11 AND ruleset=1 + } +} {abcde 0 axcde 1 abcye 10 axcye 11} +do_test fuzzer1-1.14 { + catchsql {INSERT INTO f1 VALUES(1)} +} {1 {table f1 may not be modified}} +do_test fuzzer1-1.15 { + catchsql {DELETE FROM f1} +} {1 {table f1 may not be modified}} +do_test fuzzer1-1.16 { + catchsql {UPDATE f1 SET rowid=rowid+10000} +} {1 {table f1 may not be modified}} + do_test fuzzer1-2.0 { execsql { - CREATE VIRTUAL TABLE temp.f2 USING fuzzer; -- costs based on English letter frequencies - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','e',24); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','o',47); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','u',50); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','a',23); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','i',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','o',37); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','e',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','y',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','a',41); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','e',46); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','u',57); - INSERT INTO f2(cFrom,cTo,cost) VALUES('u','o',58); - INSERT INTO f2(cFrom,cTo,cost) VALUES('y','i',33); - - INSERT INTO f2(cFrom,cTo,cost) VALUES('t','th',70); - INSERT INTO f2(cFrom,cTo,cost) VALUES('th','t',66); + CREATE TEMP TABLE f2_rules(ruleset DEFAULT 0, cFrom, cTo, cost); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','e',24); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','o',47); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','u',50); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','a',23); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','i',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','o',37); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','e',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','y',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','a',41); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','e',46); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','u',57); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('u','o',58); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('y','i',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('t','th',70); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('th','t',66); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','b',106); - INSERT INTO f2(cFrom,cTo,cost) VALUES('b','',106); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','c',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('c','',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','d',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('d','',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','e',83); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','',83); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','f',97); - INSERT INTO f2(cFrom,cTo,cost) VALUES('f','',97); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','g',99); - INSERT INTO f2(cFrom,cTo,cost) VALUES('g','',99); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','h',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('h','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','i',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','j',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('j','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','k',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('k','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','l',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('l','',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','m',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('m','',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','n',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('n','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','o',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','p',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('p','',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','q',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('q','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','r',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('r','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','s',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('s','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','t',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('t','',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','u',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('u','',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','v',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('v','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','w',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('w','',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','x',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('x','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','y',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('y','',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','z',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('z','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','b',106); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('b','',106); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','c',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('c','',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','d',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('d','',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','e',83); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','',83); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','f',97); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('f','',97); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','g',99); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('g','',99); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','h',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('h','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','i',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','j',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('j','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','k',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('k','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','l',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('l','',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','m',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('m','',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','n',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('n','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','o',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','p',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('p','',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','q',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('q','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','r',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('r','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','s',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('s','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','t',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('t','',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','u',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('u','',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','v',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('v','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','w',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('w','',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','x',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('x','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','y',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('y','',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','z',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('z','',120); + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 1, cFrom, cTo, 100 FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 2, cFrom, cTo, 200-cost FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 3, cFrom, cTo, cost FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + VALUES(3, 'mallard','duck',50), + (3, 'duck', 'mallard', 50), + (3, 'rock', 'stone', 50), + (3, 'stone', 'rock', 50); + + + CREATE VIRTUAL TABLE temp.f2 USING fuzzer(f2_rules); -- Street names for the 28269 ZIPCODE. -- @@ -1377,6 +1510,359 @@ do_test fuzzer1-2.3 { AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } } {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} +do_test fuzzer1-2.4 { + execsql { + SELECT DISTINCT streetname.n + FROM f2 JOIN streetname + ON (streetname.n>=f2.word AND streetname.n<=(f2.word || 'zzzzzz')) + WHERE f2.word MATCH 'duck' + AND f2.distance<150 + AND f2.ruleset=3 + ORDER BY 1 + } +} {mallard {mallard cove} {mallard forest} {mallard grove} {mallard hill} {mallard park} {mallard ridge} {mallard view}} +do_test fuzzer1-2.5 { + execsql { + SELECT DISTINCT streetname.n + FROM f2 JOIN streetname + ON (streetname.n>=f2.word AND streetname.n<=(f2.word || 'zzzzzz')) + WHERE f2.word MATCH 'duck' + AND f2.distance<150 + AND f2.ruleset=2 + ORDER BY 1 + } +} {} +forcedelete test.db2 +do_execsql_test fuzzer1-4.1 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.f3_rules(ruleset, cfrom, cto, cost); + INSERT INTO f3_rules(ruleset, cfrom, cto, cost) VALUES(0, 'x','y', 10); + INSERT INTO f3_rules(ruleset, cfrom, cto, cost) VALUES(1, 'a','b', 10); + CREATE VIRTUAL TABLE aux.f3 USING fuzzer(f3_rules); + SELECT word FROM f3 WHERE word MATCH 'ax' +} {ax ay} + +#------------------------------------------------------------------------- +# +# 1.5.1 - Check things work with a fuzzer data table name that requires +# quoting. Also that NULL entries in the "from" column of the +# data table are treated as zero length strings (''). +# +# 1.5.2 - Check that no-op rules (i.e. C->C) are ignored. Test NULL in +# the "to" column of a fuzzer data table. +# +# 1.5.3 - Test out-of-range values for the cost field of the data table. +# +# 1.5.4 - Test out-of-range values for the string fields of the data table. +# +# 1.5.5 - Test out-of-range values for the ruleset field of the data table. +# +do_execsql_test 5.1 { + CREATE TABLE "fuzzer [x] rules table"(a, b, c, d); + INSERT INTO "fuzzer [x] rules table" VALUES(0, NULL, 'abc', 10); + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + SELECT word, distance FROM x WHERE word MATCH '123' LIMIT 4; +} {123 0 abc123 10 1abc23 10 12abc3 10} + +do_execsql_test 5.2 { + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'x', NULL, 20); + INSERT INTO "fuzzer [x] rules table" VALUES(0, NULL, NULL, 10); + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'x', 'x', 10); + + DROP TABLE x; + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + + SELECT word, distance FROM x WHERE word MATCH 'xx'; +} {xx 0 x 20 {} 40} + +do_execsql_test 5.3.1 { + DROP TABLE IF EXISTS x; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'c', 'd', 1001); +} +do_catchsql_test 5.3.2 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.3.3 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'd', 'c', 0); +} +do_catchsql_test 5.3.4 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.3.5 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'd', 'c', -20); +} +do_catchsql_test 5.3.6 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.4.1 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, 'x', '12345678901234567890123456789012345678901234567890', 2 + ); + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + SELECT word FROM x WHERE word MATCH 'x'; +} {x 12345678901234567890123456789012345678901234567890} + +do_execsql_test 5.4.2 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, 'x', '123456789012345678901234567890123456789012345678901', 2 + ); +} +do_catchsql_test 5.4.3 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: maximum string length is 50}} + +do_execsql_test 5.4.4 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, '123456789012345678901234567890123456789012345678901', 'x', 2 + ); +} +do_catchsql_test 5.4.5 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: maximum string length is 50}} + +do_execsql_test 5.5.1 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(-1, 'x', 'y', 2); +} +do_catchsql_test 5.5.2 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: ruleset must be between 0 and 2147483647}} + +do_execsql_test 5.5.3 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES((1<<32)+100, 'x', 'y', 2); +} +do_catchsql_test 5.5.4 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: ruleset must be between 0 and 2147483647}} + +#------------------------------------------------------------------------- +# This test uses a fuzzer table with many rules. There is one rule to +# map each possible two character string, where characters are lower-case +# letters used in the English language, to all other possible two character +# strings. In total, (26^4)-(26^2) mappings (the subtracted term represents +# the no-op mappings discarded automatically by the fuzzer). +# +# +do_execsql_test 6.1.1 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x1_rules; + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); +} +puts "This test is slow - perhaps around 7 seconds on an average pc" +do_test 6.1.2 { + set LETTERS {a b c d e f g h i j k l m n o p q r s t u v w x y z} + set cost 1 + db transaction { + foreach c1 $LETTERS { + foreach c2 $LETTERS { + foreach c3 $LETTERS { + foreach c4 $LETTERS { + db eval {INSERT INTO x1_rules VALUES(0, $c1||$c2, $c3||$c4, $cost)} + set cost [expr ($cost%1000) + 1] + } + } + } + } + db eval {UPDATE x1_rules SET cost = 20 WHERE cost<20 AND cFrom!='xx'} + } +} {} + +do_execsql_test 6.2 { + SELECT count(*) FROM x1_rules WHERE cTo!=cFrom; +} [expr 26*26*26*26 - 26*26] + +do_execsql_test 6.2.1 { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT word FROM x1 WHERE word MATCH 'xx' LIMIT 10; +} {xx hw hx hy hz ia ib ic id ie} +do_execsql_test 6.2.2 { + SELECT cTo FROM x1_rules WHERE cFrom='xx' + ORDER BY cost asc, rowid asc LIMIT 9; +} {hw hx hy hz ia ib ic id ie} + +#------------------------------------------------------------------------- +# Test using different types of quotes with CREATE VIRTUAL TABLE +# arguments. +# +do_execsql_test 7.1 { + CREATE TABLE [x2 "rules] (a, b, c, d); + INSERT INTO [x2 "rules] VALUES(0, 'a', 'b', 5); +} +foreach {tn sql} { + 1 { CREATE VIRTUAL TABLE x2 USING fuzzer( [x2 "rules] ) } + 2 { CREATE VIRTUAL TABLE x2 USING fuzzer( "x2 ""rules" ) } + 3 { CREATE VIRTUAL TABLE x2 USING fuzzer( 'x2 "rules' ) } + 4 { CREATE VIRTUAL TABLE x2 USING fuzzer( `x2 "rules` ) } +} { + do_execsql_test 7.2.$tn.1 { DROP TABLE IF EXISTS x2 } + do_execsql_test 7.2.$tn.2 $sql + do_execsql_test 7.2.$tn.3 { + SELECT word FROM x2 WHERE word MATCH 'aaa' + } {aaa baa aba aab bab abb bba bbb} +} + +#------------------------------------------------------------------------- +# Test using a fuzzer table in different contexts. +# +do_execsql_test 8.1 { + CREATE TABLE x3_rules(rule_set, cFrom, cTo, cost); + INSERT INTO x3_rules VALUES(2, 'a', 'x', 10); + INSERT INTO x3_rules VALUES(2, 'a', 'y', 9); + INSERT INTO x3_rules VALUES(2, 'a', 'z', 8); + CREATE VIRTUAL TABLE x3 USING fuzzer(x3_rules); +} + +do_execsql_test 8.2.1 { + SELECT cFrom, cTo, word + FROM x3_rules CROSS JOIN x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a x x a y y a z z} + +do_execsql_test 8.2.2 { + SELECT cFrom, cTo, word + FROM x3 CROSS JOIN x3_rules + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.3 { + SELECT cFrom, cTo, word + FROM x3_rules, x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.4 { + SELECT cFrom, cTo, word + FROM x3, x3_rules + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.5 { + CREATE INDEX i1 ON x3_rules(cost); + SELECT cFrom, cTo, word + FROM x3_rules, x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.5 { + SELECT word FROM x3_rules, x3 WHERE word MATCH x3_rules.cFrom AND ruleset=2; +} {a z y x a z y x a z y x} + +do_execsql_test 8.2.6 { + SELECT word FROM x3_rules, x3 + WHERE word MATCH x3_rules.cFrom + AND ruleset=2 + AND x3_rules.cost=8; +} {a z y x} + +do_execsql_test 8.2.7 { + CREATE TABLE t1(a, b); + CREATE INDEX i2 ON t1(b); + SELECT word, distance FROM x3, t1 + WHERE x3.word MATCH t1.a AND ruleset=2 AND distance=t1.b; +} {} + +do_execsql_test 8.2.8 { + INSERT INTO x3_rules VALUES(1, 'a', 't', 5); + INSERT INTO x3_rules VALUES(1, 'a', 'u', 4); + INSERT INTO x3_rules VALUES(1, 'a', 'v', 3); + DROP TABLE x3; + CREATE VIRTUAL TABLE x3 USING fuzzer(x3_rules); + SELECT * FROM x3_rules; +} { + 2 a x 10 + 2 a y 9 + 2 a z 8 + 1 a t 5 + 1 a u 4 + 1 a v 3 +} + +do_catchsql_test 8.2.9 { + SELECT word FROM x3 WHERE ruleset=2 AND word MATCH 'a' AND WORD MATCH 'b'; +} {1 {unable to use function MATCH in the requested context}} + +do_execsql_test 8.2.10 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' +} {a v u t} + +# The term "ruleset<=1" is not handled by the fuzzer module. Instead, it +# is handled by SQLite, which assumes that all rows have a NULL value in +# the ruleset column. Since NULL<=1 is never true, this query returns +# no rows. +do_execsql_test 8.2.11 { + SELECT word FROM x3 WHERE ruleset<=1 AND word MATCH 'a' +} {} + +do_execsql_test 8.2.12 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY distance ASC; +} {a v u t} + +do_execsql_test 8.2.13 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY distance DESC; +} {t u v a} + +do_execsql_test 8.2.13 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY word ASC; +} {a t u v} + +do_execsql_test 8.2.14 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY word DESC; +} {v u t a} + +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE x4_rules(a, b, c, d); + INSERT INTO x4_rules VALUES(0, 'a', 'b', 10); + INSERT INTO x4_rules VALUES(0, 'a', 'c', 11); + INSERT INTO x4_rules VALUES(0, 'bx', 'zz', 20); + INSERT INTO x4_rules VALUES(0, 'cx', 'yy', 15); + INSERT INTO x4_rules VALUES(0, 'zz', '!!', 50); + CREATE VIRTUAL TABLE x4 USING fuzzer(x4_rules); +} + +do_execsql_test 9.2 { + SELECT word, distance FROM x4 WHERE word MATCH 'ax'; +} {ax 0 bx 10 cx 11 yy 26 zz 30 !! 80} + + +do_execsql_test 10.1 { + CREATE TABLE x5_rules(a, b, c, d); + CREATE VIRTUAL TABLE x5 USING fuzzer(x5_rules); +} + +do_execsql_test 10.2 { + SELECT word, distance FROM x5 WHERE word MATCH + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' || + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' || + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' +} {} + +do_execsql_test 10.3 { + INSERT INTO x5_rules VALUES(0, 'a', '0.1.2.3.4.5.6.7.8.9.a', 1); + DROP TABLE x5; + CREATE VIRTUAL TABLE x5 USING fuzzer(x5_rules); + SELECT length(word) FROM x5 WHERE word MATCH 'a' LIMIT 50; +} {1 21 41 61 81} finish_test + + diff --git a/test/fuzzerfault.test b/test/fuzzerfault.test new file mode 100644 index 0000000000..067da7f5fc --- /dev/null +++ b/test/fuzzerfault.test @@ -0,0 +1,92 @@ +# 2012 February 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for TCL interface to the +# SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !vtab { finish_test ; return } +set ::testprefix fuzzerfault + +register_fuzzer_module db + +do_test 1-pre1 { + execsql { + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x1_rules VALUES(0, 'a', 'b', 1); + INSERT INTO x1_rules VALUES(0, 'a', 'c', 2); + INSERT INTO x1_rules VALUES(0, 'a', 'd', 3); + } + faultsim_save_and_close +} {} +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT word FROM x1 WHERE word MATCH 'xax'; + } +} -test { + faultsim_test_result {0 {xax xbx xcx xdx}} \ + {1 {vtable constructor failed: x1}} +} + +do_test 2-pre1 { + faultsim_delete_and_reopen + register_fuzzer_module db + execsql { + CREATE TABLE x2_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x2_rules VALUES(0, 'a', 'x', 1); + INSERT INTO x2_rules VALUES(0, 'b', 'x', 2); + INSERT INTO x2_rules VALUES(0, 'c', 'x', 3); + CREATE VIRTUAL TABLE x2 USING fuzzer(x2_rules); + } + faultsim_save_and_close +} {} + +do_faultsim_test 2 -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + SELECT count(*) FROM x2 WHERE word MATCH 'abc'; + } +} -test { + faultsim_test_result {0 8} {1 {vtable constructor failed: x2}} +} + +do_test 3-pre1 { + faultsim_delete_and_reopen + execsql { + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x1_rules VALUES(0, 'a', + '123456789012345678901234567890a1234567890123456789', 10 + ); + } + faultsim_save_and_close +} {} + +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT count(*) FROM (SELECT * FROM x1 WHERE word MATCH 'a' LIMIT 2); + } +} -test { + faultsim_test_result {0 2} {1 {vtable constructor failed: x1}} +} + + +finish_test diff --git a/test/in.test b/test/in.test index 2c38a0f4ca..108f35589c 100644 --- a/test/in.test +++ b/test/in.test @@ -431,6 +431,7 @@ do_test in-12.9 { } {1 {SELECTs to the left and right of INTERSECT do not have the same number of result columns}} } +ifcapable compound { do_test in-12.10 { catchsql { SELECT * FROM t2 WHERE a IN ( @@ -459,6 +460,7 @@ do_test in-12.13 { ); } } {1 {only a single result allowed for a SELECT that is part of an expression}} +}; #ifcapable compound #------------------------------------------------------------------------ diff --git a/test/incrblob.test b/test/incrblob.test index 388c4ba3a3..1880128f83 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -473,15 +473,9 @@ if {[permutation] != "memsubsys1"} { flush $::blob } {} - # At this point rollback should be illegal (because - # there is an open blob channel). But commit is also illegal because - # the open blob is read-write. + # At this point commit should be illegal (because + # there is an open blob channel). # - do_test incrblob-6.10 { - catchsql { - ROLLBACK; - } db2 - } {1 {cannot rollback transaction - SQL statements in progress}} do_test incrblob-6.11 { catchsql { COMMIT; diff --git a/test/insert.test b/test/insert.test index 05718d556f..e00b9a8028 100644 --- a/test/insert.test +++ b/test/insert.test @@ -388,18 +388,20 @@ do_test insert-9.2 { # Multiple VALUES clauses # -do_test insert-10.1 { - execsql { - CREATE TABLE t10(a,b,c); - INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); - SELECT * FROM t10; - } -} {1 2 3 4 5 6 7 8 9} -do_test insert-10.2 { - catchsql { - INSERT INTO t10 VALUES(11,12,13), (14,15); - } -} {1 {all VALUES must have the same number of terms}} +ifcapable compound { + do_test insert-10.1 { + execsql { + CREATE TABLE t10(a,b,c); + INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); + SELECT * FROM t10; + } + } {1 2 3 4 5 6 7 8 9} + do_test insert-10.2 { + catchsql { + INSERT INTO t10 VALUES(11,12,13), (14,15); + } + } {1 {all VALUES must have the same number of terms}} +} integrity_check insert-99.0 diff --git a/test/join6.test b/test/join6.test index 4f65dcb446..7fbf508e57 100644 --- a/test/join6.test +++ b/test/join6.test @@ -124,26 +124,28 @@ do_test join6-3.6 { } } {1 91 92 3 93 5 91 2 93 94 4 95 6 99} -do_test join6-4.1 { - execsql { - SELECT * FROM - (SELECT 1 AS a, 91 AS x, 92 AS y UNION SELECT 2, 93, 94) - NATURAL JOIN t2 NATURAL JOIN t3 - } -} {1 91 92 3 93 5} -do_test join6-4.2 { - execsql { - SELECT * FROM t1 NATURAL JOIN - (SELECT 3 AS b, 92 AS y, 93 AS z UNION SELECT 4, 94, 95) - NATURAL JOIN t3 - } -} {1 91 92 3 93 5} -do_test join6-4.3 { - execsql { - SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN - (SELECT 5 AS c, 91 AS x, 93 AS z UNION SELECT 6, 99, 95) - } -} {1 91 92 3 93 5} +ifcapable compound { + do_test join6-4.1 { + execsql { + SELECT * FROM + (SELECT 1 AS a, 91 AS x, 92 AS y UNION SELECT 2, 93, 94) + NATURAL JOIN t2 NATURAL JOIN t3 + } + } {1 91 92 3 93 5} + do_test join6-4.2 { + execsql { + SELECT * FROM t1 NATURAL JOIN + (SELECT 3 AS b, 92 AS y, 93 AS z UNION SELECT 4, 94, 95) + NATURAL JOIN t3 + } + } {1 91 92 3 93 5} + do_test join6-4.3 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN + (SELECT 5 AS c, 91 AS x, 93 AS z UNION SELECT 6, 99, 95) + } + } {1 91 92 3 93 5} +} diff --git a/test/journal3.test b/test/journal3.test index f1bf89f8c7..939cc27c70 100644 --- a/test/journal3.test +++ b/test/journal3.test @@ -22,7 +22,9 @@ source $testdir/malloc_common.tcl # if {$::tcl_platform(platform) == "unix"} { - set umask [exec /bin/sh -c umask] + # Changed on 2012-02-13: umask is deliberately ignored for -wal, -journal, + # and -shm files. + #set umask [exec /bin/sh -c umask] faultsim_delete_and_reopen do_test journal3-1.1 { execsql { CREATE TABLE tx(y, z) } } {} @@ -33,7 +35,8 @@ if {$::tcl_platform(platform) == "unix"} { 4 00755 } { db close - set effective [format %.5o [expr $permissions & ~$umask]] + #set effective [format %.5o [expr $permissions & ~$umask]] + set effective $permissions do_test journal3-1.2.$tn.1 { catch { forcedelete test.db-journal } file attributes test.db -permissions $permissions diff --git a/test/minmax4.test b/test/minmax4.test index 12f56a86c3..0d8305b5ff 100644 --- a/test/minmax4.test +++ b/test/minmax4.test @@ -20,6 +20,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test minmax4-1.1 { db eval { CREATE TABLE t1(p,q); diff --git a/test/multiplex3.test b/test/multiplex3.test index 0f52d763be..c1e741acdb 100644 --- a/test/multiplex3.test +++ b/test/multiplex3.test @@ -127,6 +127,39 @@ for {set iTest 1} {$iTest<=100} {incr iTest} { db2 close } +catch { db close } + + +do_test 3.0 { setup_and_save_db } {} +do_faultsim_test 3 -faults ioerr-trans* -prep { + + forcedelete test2.db + set fd [open test2.wal w] + seek $fd 4095 + puts -nonewline $fd x + close $fd + + multiplex_restore_db + sqlite3 db file:test.db?8_3_names=1 + sqlite3 db2 file:test2.db?8_3_names=1 + sqlite3_multiplex_control db main chunk_size [expr 256*1024] + sqlite3_multiplex_control db2 main chunk_size [expr 256*1024] +} -body { + sqlite3_backup B db2 main db main + B step 100000 + set rc [B finish] + if { [string match SQLITE_IOERR_* $rc] } {error "disk I/O error"} + set rc +} -test { + faultsim_test_result {0 SQLITE_OK} + if {$testrc==0} { + set cksum2 [execsql {SELECT md5sum(a, b) FROM t1 ORDER BY a} db2] + if {$cksum2 != $::cksum1} { error "data mismatch" } + } + catch { B finish } + catch { db close } + catch { db2 close } +} catch { db close } sqlite3_multiplex_shutdown diff --git a/test/permutations.test b/test/permutations.test index 19388b1844..d7ce9cfe3d 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -187,8 +187,7 @@ test_suite "fts3" -prefix "" -description { fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test - fts3corrupt2.test - fts3first.test + fts3corrupt2.test fts3first.test fts4langid.test } diff --git a/test/pragma.test b/test/pragma.test index cbe35ea806..3f17bd742b 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1489,4 +1489,26 @@ foreach {temp_setting val} { } $val } +# The SQLITE_FCNTL_PRAGMA logic, with error handling. +# +db close +testvfs tvfs +sqlite3 db test.db -vfs tvfs +do_test pragma-19.1 { + catchsql {PRAGMA error} +} {1 {SQL logic error or missing database}} +do_test pragma-19.2 { + catchsql {PRAGMA error='This is the error message'} +} {1 {This is the error message}} +do_test pragma-19.3 { + catchsql {PRAGMA error='7 This is the error message'} +} {1 {This is the error message}} +do_test pragma-19.4 { + catchsql {PRAGMA error=7} +} {1 {out of memory}} +do_test pragma-19.5 { + file tail [lindex [execsql {PRAGMA filename}] 0] +} {test.db} + + finish_test diff --git a/test/randexpr1.test b/test/randexpr1.test index 10a1d17c39..7a98f0be0f 100644 --- a/test/randexpr1.test +++ b/test/randexpr1.test @@ -22,6 +22,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Create test data # do_test randexpr1-1.1 { diff --git a/test/savepoint.test b/test/savepoint.test index 3bbbaaae31..015d97fe1a 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -303,11 +303,19 @@ ifcapable incrblob { execsql {SAVEPOINT abc} catchsql {ROLLBACK TO def} } {1 {no such savepoint: def}} - do_test savepoint-5.3.2 { + do_test savepoint-5.3.2.1 { execsql {SAVEPOINT def} set fd [db incrblob -readonly blobs x 1] + set rc [catch {seek $fd 0;read $fd} res] + lappend rc $res + } {0 {hellontyeight character blob}} + do_test savepoint-5.3.2.2 { catchsql {ROLLBACK TO def} - } {1 {cannot rollback savepoint - SQL statements in progress}} + } {0 {}} + do_test savepoint-5.3.2.3 { + set rc [catch {seek $fd 0; read $fd} res] + set rc + } {1} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} @@ -649,10 +657,8 @@ if {[wal_is_wal_mode]==0} { CREATE TABLE main.t1(x, y); CREATE TABLE aux1.t2(x, y); CREATE TABLE aux2.t3(x, y); - SELECT name FROM sqlite_master - UNION ALL - SELECT name FROM aux1.sqlite_master - UNION ALL + SELECT name FROM sqlite_master; + SELECT name FROM aux1.sqlite_master; SELECT name FROM aux2.sqlite_master; } } {t1 t2 t3} @@ -691,7 +697,7 @@ if {[wal_is_wal_mode]==0} { execsql { PRAGMA lock_status } } [list main reserved temp $templockstate aux1 reserved aux2 reserved] do_test savepoint-10.2.9 { - execsql { SELECT 'a', * FROM t1 UNION ALL SELECT 'b', * FROM t3 } + execsql { SELECT 'a', * FROM t1 ; SELECT 'b', * FROM t3 } } {a 1 2 b 3 4} do_test savepoint-10.2.9 { execsql { diff --git a/test/select1.test b/test/select1.test index 73b0e40fe4..852b52e284 100644 --- a/test/select1.test +++ b/test/select1.test @@ -1066,5 +1066,11 @@ if {[db one {PRAGMA locking_mode}]=="normal"} { execsql { SELECT 2 IN (SELECT a FROM t1) } } {1} } + +# Crash bug reported on the mailing list on 2012-02-23 +# +do_test select1-16.1 { + catchsql {SELECT 1 FROM (SELECT *)} +} {1 {no tables specified}} finish_test diff --git a/test/select4.test b/test/select4.test index dff0b90fee..e205b37a4f 100644 --- a/test/select4.test +++ b/test/select4.test @@ -805,4 +805,23 @@ do_test select4-12.1 { } ;# ifcapable compound + +# Ticket [3557ad65a076c] - Incorrect DISTINCT processing with an +# indexed query using IN. +# +do_test select4-13.1 { + sqlite3 db test.db + db eval { + CREATE TABLE t13(a,b); + INSERT INTO t13 VALUES(1,1); + INSERT INTO t13 VALUES(2,1); + INSERT INTO t13 VALUES(3,1); + INSERT INTO t13 VALUES(2,2); + INSERT INTO t13 VALUES(3,2); + INSERT INTO t13 VALUES(4,2); + CREATE INDEX t13ab ON t13(a,b); + SELECT DISTINCT b from t13 WHERE a IN (1,2,3); + } +} {1 2} + finish_test diff --git a/test/selectC.test b/test/selectC.test index fd384053ce..dedac41fc4 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -151,7 +151,7 @@ do_test selectC-1.14.2 { # The following query used to leak memory. Verify that has been fixed. # -ifcapable trigger { +ifcapable trigger&&compound { do_test selectC-2.1 { catchsql { CREATE TABLE t21a(a,b); diff --git a/test/shared2.test b/test/shared2.test index d40c9a271a..5bde8cfad8 100644 --- a/test/shared2.test +++ b/test/shared2.test @@ -79,48 +79,6 @@ do_test shared2-1.3 { list $a $count } {32 64} -#--------------------------------------------------------------------------- -# These tests, shared2.2.*, test the outcome when data is added to or -# removed from a table due to a rollback while a read-uncommitted -# cursor is scanning it. -# -do_test shared2-2.1 { - execsql { - INSERT INTO numbers VALUES(1, 'Medium length text field'); - INSERT INTO numbers VALUES(2, 'Medium length text field'); - INSERT INTO numbers VALUES(3, 'Medium length text field'); - INSERT INTO numbers VALUES(4, 'Medium length text field'); - BEGIN; - DELETE FROM numbers WHERE (a%2)=0; - } db1 - set res [list] - db2 eval { - SELECT a FROM numbers ORDER BY a; - } { - lappend res $a - if {$a==3} { - execsql {ROLLBACK} db1 - } - } - set res -} {1 3 4} -do_test shared2-2.2 { - execsql { - BEGIN; - INSERT INTO numbers VALUES(5, 'Medium length text field'); - INSERT INTO numbers VALUES(6, 'Medium length text field'); - } db1 - set res [list] - db2 eval { - SELECT a FROM numbers ORDER BY a; - } { - lappend res $a - if {$a==5} { - execsql {ROLLBACK} db1 - } - } - set res -} {1 2 3 4 5} db1 close db2 close diff --git a/test/stat.test b/test/stat.test index 21726eb258..926d9b7406 100644 --- a/test/stat.test +++ b/test/stat.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !vtab { +ifcapable !vtab||!compound { finish_test return } diff --git a/test/syscall.test b/test/syscall.test index dde4b467a2..d841a9a0b2 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs + statvfs fchown umask } { if {[test_syscall exists $s]} {lappend syscall_list $s} } diff --git a/test/tkt-02a8e81d44.test b/test/tkt-02a8e81d44.test index 4a48fb029e..7ca9866414 100644 --- a/test/tkt-02a8e81d44.test +++ b/test/tkt-02a8e81d44.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-02a838-1.1 { execsql { CREATE TABLE t1(a); diff --git a/test/tkt-38cb5df375.test b/test/tkt-38cb5df375.test index 47b0b55106..e5e02671b7 100644 --- a/test/tkt-38cb5df375.test +++ b/test/tkt-38cb5df375.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-38cb5df375.0 { execsql { CREATE TABLE t1(a); diff --git a/test/tkt-3a77c9714e.test b/test/tkt-3a77c9714e.test index d77ee42bb3..6eaec16cdd 100644 --- a/test/tkt-3a77c9714e.test +++ b/test/tkt-3a77c9714e.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix "tkt-3a77c9714e" do_execsql_test 1.1 { diff --git a/test/tkt-7bbfb7d442.test b/test/tkt-7bbfb7d442.test index e560a0d4d5..dcb9b16058 100644 --- a/test/tkt-7bbfb7d442.test +++ b/test/tkt-7bbfb7d442.test @@ -49,10 +49,12 @@ do_execsql_test 1.2 { do_execsql_test 1.3 { DELETE FROM t3 } -do_execsql_test 1.4 { - INSERT INTO t3(t3_a) SELECT 1 UNION SELECT 2 UNION SELECT 3; - SELECT * FROM t3; -} {1 I 2 II 3 III} +ifcapable compound { + do_execsql_test 1.4 { + INSERT INTO t3(t3_a) SELECT 1 UNION SELECT 2 UNION SELECT 3; + SELECT * FROM t3; + } {1 I 2 II 3 III} +} diff --git a/test/tkt-80ba201079.test b/test/tkt-80ba201079.test index 95e99b5a5b..0122e95f2f 100644 --- a/test/tkt-80ba201079.test +++ b/test/tkt-80ba201079.test @@ -164,11 +164,13 @@ do_execsql_test 303 { (b='B' AND c IN (SELECT c FROM t1)) } {A B C D E} -do_execsql_test 304 { - SELECT * FROM t1, t2 WHERE - (a='A' AND d='E') OR - (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) -} {A B C D E} +ifcapable compound { + do_execsql_test 304 { + SELECT * FROM t1, t2 WHERE + (a='A' AND d='E') OR + (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) + } {A B C D E} +} do_execsql_test 305 { SELECT * FROM t1, t2 WHERE @@ -182,10 +184,12 @@ do_execsql_test 306 { (a='A' AND d='E') } {A B C D E} -do_execsql_test 307 { - SELECT * FROM t1, t2 WHERE - (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) OR - (a='A' AND d='E') -} {A B C D E} +ifcapable compound { + do_execsql_test 307 { + SELECT * FROM t1, t2 WHERE + (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) OR + (a='A' AND d='E') + } {A B C D E} +} finish_test diff --git a/test/tkt-b72787b1.test b/test/tkt-b72787b1.test index 11ea41ed7a..dea3f49a64 100644 --- a/test/tkt-b72787b1.test +++ b/test/tkt-b72787b1.test @@ -35,6 +35,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + unset -nocomplain ::STMT proc runsql {} { db eval {CREATE TABLE IF NOT EXISTS t4(q)} diff --git a/test/tkt-d82e3f3721.test b/test/tkt-d82e3f3721.test index 31f7d342c7..da932d1eaa 100644 --- a/test/tkt-d82e3f3721.test +++ b/test/tkt-d82e3f3721.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-d82e3-1.1 { db eval { CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index 6f0b43fa8e..af6f71ad96 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-f7772-1.1 { execsql { CREATE TEMP TABLE t1(x UNIQUE); @@ -37,7 +42,7 @@ do_test tkt-f7772-1.2 { BEGIN IMMEDIATE; SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-1.3 { sqlite3_get_autocommit db } {1} diff --git a/test/tkt3527.test b/test/tkt3527.test index 34e9e6121f..d9b1dadfdc 100644 --- a/test/tkt3527.test +++ b/test/tkt3527.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt3527-1.1 { db eval { CREATE TABLE Element ( diff --git a/test/tkt3773.test b/test/tkt3773.test index 0dc414e50e..3f5a1a3217 100644 --- a/test/tkt3773.test +++ b/test/tkt3773.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt3773-1.1 { db eval { CREATE TABLE t1(a,b); diff --git a/test/trans3.test b/test/trans3.test index ab7db6a277..d5b316bec8 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -64,14 +64,13 @@ do_test trans3-1.5 { } } errmsg] lappend x $errmsg -} {1 {cannot rollback transaction - SQL statements in progress}} +} {1 {abort due to ROLLBACK}} do_test trans3-1.6 { set ::ecode -} {SQLITE_BUSY} +} {} do_test trans3-1.7 { - db eval COMMIT db eval {SELECT * FROM t1} -} {1 2 3 4 5} +} {1 2 3 4} unset -nocomplain ecode finish_test diff --git a/test/trigger1.test b/test/trigger1.test index 1e77fe0fc8..9d917bdb38 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -29,7 +29,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable {!trigger} { +ifcapable !trigger||!compound { finish_test return } diff --git a/test/vtabD.test b/test/vtabD.test index 509ba45745..589f518983 100644 --- a/test/vtabD.test +++ b/test/vtabD.test @@ -49,17 +49,15 @@ do_test vtabD-1.5 { do_test vtabD-1.6 { execsql { SELECT * FROM tv1 WHERE a < 500 OR b = 810000 } } [execsql { - SELECT * FROM t1 WHERE a < 500 - UNION ALL - SELECT * FROM t1 WHERE b = 810000 AND NOT (a < 500) + SELECT * FROM t1 WHERE a < 500; + SELECT * FROM t1 WHERE b = 810000 AND NOT (a < 500); }] do_test vtabD-1.7 { execsql { SELECT * FROM tv1 WHERE a < 90000 OR b = 8100000000 } } [execsql { - SELECT * FROM t1 WHERE a < 90000 - UNION ALL - SELECT * FROM t1 WHERE b = 8100000000 AND NOT (a < 90000) + SELECT * FROM t1 WHERE a < 90000; + SELECT * FROM t1 WHERE b = 8100000000 AND NOT (a < 90000); }] if {[working_64bit_int]} { diff --git a/test/vtab_shared.test b/test/vtab_shared.test index ce2e432fcf..6a76e2749b 100644 --- a/test/vtab_shared.test +++ b/test/vtab_shared.test @@ -124,23 +124,25 @@ breakpoint execsql { SELECT * FROM t3 } db2 } {1 2 3 4 5 6} -do_test vtab_shared-1.12.1 { - db close - execsql { - SELECT * FROM t1 UNION ALL - SELECT * FROM t2 UNION ALL - SELECT * FROM t3 - } db2 -} {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} -do_test vtab_shared-1.12.2 { - sqlite3 db test.db - register_echo_module [sqlite3_connection_pointer db] - execsql { - SELECT * FROM t1 UNION ALL - SELECT * FROM t2 UNION ALL - SELECT * FROM t3 - } db -} {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} +ifcapable compound { + do_test vtab_shared-1.12.1 { + db close + execsql { + SELECT * FROM t1 UNION ALL + SELECT * FROM t2 UNION ALL + SELECT * FROM t3 + } db2 + } {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} + do_test vtab_shared-1.12.2 { + sqlite3 db test.db + register_echo_module [sqlite3_connection_pointer db] + execsql { + SELECT * FROM t1 UNION ALL + SELECT * FROM t2 UNION ALL + SELECT * FROM t3 + } db + } {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} +} # Try a rename or two. # diff --git a/test/wal2.test b/test/wal2.test index c7f00ea60f..cbefb7ab1e 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1042,7 +1042,10 @@ tvfs delete # if {$::tcl_platform(platform) == "unix"} { faultsim_delete_and_reopen - set umask [exec /bin/sh -c umask] + # Changed on 2012-02-13: umask is deliberately ignored for -wal files. + #set umask [exec /bin/sh -c umask] + set umask 0 + do_test wal2-12.1 { sqlite3 db test.db diff --git a/test/wal8.test b/test/wal8.test new file mode 100644 index 0000000000..4b97de7ae7 --- /dev/null +++ b/test/wal8.test @@ -0,0 +1,90 @@ +# 2012 February 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# +# Specifically, it tests the case where a connection opens an empty +# file. Then, another connection opens the same file and initializes +# the connection as a WAL database. Following this, the first connection +# executes a "PRAGMA page_size = XXX" command to set its expected page +# size, and then queries the database. +# +# This is an unusual case, as normally SQLite is able to glean the page +# size from the database file as soon as it is opened (even before the +# first read transaction is executed), and the "PRAGMA page_size = XXX" +# is a no-op. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix wal8 + +db close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 1.0 { + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } db2 +} {wal} + +do_catchsql_test 1.1 { + PRAGMA page_size = 4096; + VACUUM; +} {0 {}} + +db close +db2 close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 2.0 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + PRAGMA journal_mode = wal; + } db2 +} {wal} + +do_catchsql_test 2.1 { + PRAGMA page_size = 4096; + VACUUM; +} {0 {}} + +db close +db2 close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 3.0 { + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } db2 +} {wal} + +do_execsql_test 3.1 { + PRAGMA page_size = 4096; + SELECT name FROM sqlite_master; +} {t1} + +finish_test + diff --git a/test/where9.test b/test/where9.test index b4a2d8d130..8c5c96e2a8 100644 --- a/test/where9.test +++ b/test/where9.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !or_opt { +ifcapable !or_opt||!compound { finish_test return }