diff --git a/Makefile.in b/Makefile.in index 2606b5a654..e4687626f0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1032,7 +1032,7 @@ FTS5_SRC = \ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h - ./lemon $(OPTS) fts5parse.y + ./lemon$(BEXE) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c @@ -1164,8 +1164,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # +VALIDIDS=' sqlite3(changeset|changegroup|session)?_' checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0 + nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index 018296b8dd..a3efad0d44 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -24,12 +24,22 @@ */ void usage(const char *zArgv0){ fprintf(stderr, -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n" +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" "\n" -" Argument RBU-DB must be an RBU database containing an update suitable for\n" -" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" -" (the default value), this program attempts to apply the entire update to\n" -" the target database.\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -vacuum\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" "\n" " If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" " to sqlite3rbu_step(). If the RBU update has not been completely applied\n" @@ -69,29 +79,43 @@ int main(int argc, char **argv){ char *zErrmsg; /* Error message, if any */ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ + int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; + int nArg = argc-2; - /* Process command line arguments. Following this block local variables - ** zTarget, zRbu and nStep are all set. */ - if( argc==5 ){ - size_t nArg1 = strlen(argv[1]); - if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); - nStep = atoi(argv[2]); - }else if( argc!=3 ){ - usage(argv[0]); + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && izTarget==0) + /************************************************************************* ** The following three functions, found below: @@ -850,8 +863,11 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ /* ** The implementation of the rbu_target_name() SQL function. This function -** accepts one argument - the name of a table in the RBU database. If the -** table name matches the pattern: +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: ** ** data[0-9]_ ** @@ -862,21 +878,33 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). */ static void rbuTargetNameFunc( - sqlite3_context *context, + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ + sqlite3rbu *p = sqlite3_user_data(pCtx); const char *zIn; - assert( argc==1 ); + assert( argc==1 || argc==2 ); zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } } } } @@ -894,7 +922,8 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " "ORDER BY name" ); @@ -1270,6 +1299,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ pStmt = 0; if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; @@ -1409,6 +1439,8 @@ static char *rbuObjIterGetIndexCols( for(i=0; pIter->abTblPk[i]==0; i++); assert( inTblCol ); zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; }else{ zCol = "rbu_rowid"; } @@ -1949,7 +1981,7 @@ static int rbuObjIterPrepareAll( } /* And to delete index entries */ - if( p->rc==SQLITE_OK ){ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError( p->dbMain, &pIter->pDelete, &p->zErrmsg, sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) @@ -1959,6 +1991,15 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", @@ -1985,7 +2026,9 @@ static int rbuObjIterPrepareAll( sqlite3_free(zWhere); sqlite3_free(zBind); }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ const char *zWrite; /* Imposter table name */ @@ -2012,8 +2055,10 @@ static int rbuObjIterPrepareAll( ); } - /* Create the DELETE statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere @@ -2021,7 +2066,7 @@ static int rbuObjIterPrepareAll( ); } - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ const char *zRbuRowid = ""; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; @@ -2071,10 +2116,16 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM '%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, pIter->zDataTbl, zLimit ) ); @@ -2169,11 +2220,15 @@ static int rbuGetUpdateStmt( return p->rc; } -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ sqlite3 *db = 0; if( p->rc==SQLITE_OK ){ const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); sqlite3_close(db); @@ -2183,6 +2238,95 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ return db; } +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. @@ -2190,10 +2334,14 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ static void rbuOpenDatabase(sqlite3rbu *p){ assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); + assert( rbuIsVacuum(p) || p->zTarget!=0 ); - p->eStage = 0; - p->dbMain = rbuOpenDbhandle(p, p->zTarget); - p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + } /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ @@ -2204,6 +2352,96 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 @@ -2218,7 +2456,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -2477,9 +2715,15 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); + const char *zMove = zBase; + char *zOal; + char *zWal; - char *zWal = sqlite3_mprintf("%s-wal", zBase); - char *zOal = sqlite3_mprintf("%s-oal", zBase); + if( rbuIsVacuum(p) ){ + zMove = sqlite3_db_filename(p->dbRbu, "main"); + } + zOal = sqlite3_mprintf("%s-oal", zMove); + zWal = sqlite3_mprintf("%s-wal", zMove); assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2500,8 +2744,8 @@ static void rbuMoveOalFile(sqlite3rbu *p){ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); p->dbMain = 0; p->dbRbu = 0; @@ -2663,19 +2907,24 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ p->rc = sqlite3_bind_value(pWriter, i+1, pVal); if( p->rc ) return; } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT , rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + if( pIter->zIdx==0 ){ + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT , rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, + rbuIsVacuum(p) ? "rowid" : "rbu_rowid" + ); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } } if( p->rc==SQLITE_OK ){ sqlite3_step(pWriter); @@ -2754,13 +3003,18 @@ static int rbuStep(sqlite3rbu *p){ /* ** Increment the schema cookie of the main database opened by p->dbMain. +** +** Or, if this is an RBU vacuum, set the schema cookie of the main db +** opened by p->dbMain to one more than the schema cookie of the main +** db opened by p->dbRbu. */ static void rbuIncrSchemaCookie(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ + sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -2788,6 +3042,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); @@ -2810,7 +3065,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, - RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, + RBU_STATE_COOKIE, (i64)pFd->iCookie, RBU_STATE_OALSZ, p->iOalSz, RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) @@ -2826,6 +3081,92 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + +/* +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. +*/ +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + + assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + /* ** Step the RBU object. */ @@ -2834,13 +3175,22 @@ int sqlite3rbu_step(sqlite3rbu *p){ switch( p->eStage ){ case RBU_STAGE_OAL: { RbuObjIter *pIter = &p->objiter; + + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){ + rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); + } + while( p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); @@ -2927,94 +3277,6 @@ int sqlite3rbu_step(sqlite3rbu *p){ } } -/* -** Free an RbuState object allocated by rbuLoadState(). -*/ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} - -/* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). -** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. -*/ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; -} - /* ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero ** otherwise. Either or both argument may be NULL. Two NULL values are @@ -3204,16 +3466,14 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } } -/* -** Open and return a new RBU handle. -*/ -sqlite3rbu *sqlite3rbu_open( + +static sqlite3rbu *openRbuHandle( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; - size_t nTarget = strlen(zTarget); + size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); size_t nState = zState ? strlen(zState) : 0; size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; @@ -3226,22 +3486,24 @@ sqlite3rbu *sqlite3rbu_open( memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); - /* Open the target database */ + /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zRbu = &p->zTarget[nTarget+1]; + char *pCsr = (char*)&p[1]; + if( zTarget ){ + p->zTarget = pCsr; + memcpy(p->zTarget, zTarget, nTarget+1); + pCsr += nTarget+1; + } + p->zRbu = pCsr; memcpy(p->zRbu, zRbu, nRbu+1); + pCsr += nRbu+1; if( zState ){ - p->zState = &p->zRbu[nRbu+1]; + p->zState = pCsr; memcpy(p->zState, zState, nState+1); } rbuOpenDatabase(p); } - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -3271,27 +3533,39 @@ sqlite3rbu *sqlite3rbu_open( } } - if( p->rc==SQLITE_OK + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) - && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie - ){ - /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on - ** page 1 of the database file. */ - p->rc = SQLITE_BUSY; - p->zErrmsg = sqlite3_mprintf("database modified during rbu update"); + && pState->eStage!=0 + ){ + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); + if( pFd->iCookie!=pState->iCookie ){ + /* At this point (pTargetFd->iCookie) contains the value of the + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on + ** page 1 of the database file. */ + p->rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", + (rbuIsVacuum(p) ? "vacuum" : "update") + ); + } } if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; + if( pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } + /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } /* Check if the main database is a zipvfs db. If it is, set the upper @@ -3336,6 +3610,28 @@ sqlite3rbu *sqlite3rbu_open( return p; } +/* +** Open and return a new RBU handle. +*/ +sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +){ + /* TODO: Check that zTarget and zRbu are non-NULL */ + return openRbuHandle(zTarget, zRbu, zState); +} + +/* +** Open a handle to begin or resume an RBU VACUUM operation. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +){ + /* TODO: Check that both arguments are non-NULL */ + return openRbuHandle(0, zTarget, zState); +} /* ** Return the database handle used by pRbu. @@ -3390,9 +3686,19 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close any open statement handles. */ rbuObjIterFinalize(&p->objiter); + /* If this is an RBU vacuum handle and the vacuum has either finished + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() + ** specifying the current target and state databases to start a new + ** vacuum from scratch. */ + if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ + int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0); + if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2; + } + /* Close the open database handle and VFS object. */ - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3594,6 +3900,22 @@ static u32 rbuGetU32(u8 *aBuf){ + ((u32)aBuf[3]); } +/* +** Write an unsigned 32-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU32(u8 *aBuf, u32 iVal){ + aBuf[0] = (iVal >> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3619,6 +3941,35 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( pRbu && rbuIsVacuum(pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + && pRbu->rc==SQLITE_OK + ){ + sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0; + rbuPutU32(&aBuf[52], iRoot); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ + + if( iAmt>100 ){ + memset(&aBuf[100], 0, iAmt-100); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + aBuf[100] = 0x0D; + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3693,7 +4044,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3705,7 +4069,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3768,6 +4134,12 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + sqlite3rbu *pRbu = (sqlite3rbu*)pArg; + pRbu->nRbu++; + pRbu->pRbuFd = p; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ @@ -3931,6 +4303,33 @@ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ return pDb; } +/* +** A main database named zName has just been opened. The following +** function returns a pointer to a buffer owned by SQLite that contains +** the name of the *-wal file this db connection will use. SQLite +** happens to pass a pointer to this buffer when using xAccess() +** or xOpen() to operate on the *-wal file. +*/ +static const char *rbuMainToWal(const char *zName, int flags){ + int n = (int)strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + return z; +} + /* ** Open an rbu file handle. */ @@ -3966,6 +4365,7 @@ static int rbuVfsOpen( rbu_file *pFd = (rbu_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; + int oflags = flags; memset(pFd, 0, sizeof(rbu_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; @@ -3978,23 +4378,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - pFd->zWal = z; + pFd->zWal = rbuMainToWal(zName, flags); } else if( flags & SQLITE_OPEN_WAL ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); @@ -4004,10 +4388,17 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - size_t nCopy = strlen(zName); - char *zCopy = sqlite3_malloc64(nCopy+2); + const char *zBase = zName; + size_t nCopy; + char *zCopy; + if( rbuIsVacuum(pDb->pRbu) ){ + zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); + } + nCopy = strlen(zBase); + zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ - memcpy(zCopy, zName, nCopy); + memcpy(zCopy, zBase, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; @@ -4022,8 +4413,17 @@ static int rbuVfsOpen( } } + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) + ){ + assert( oflags & SQLITE_OPEN_MAIN_DB ); + oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + zOpen = 0; + } + if( rc==SQLITE_OK ){ - rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f379bb5b41..fce9e493db 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -314,6 +314,38 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ); +/* +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function, which may not be NULL, identifies +** a database in which to store the state of the RBU vacuum operation if +** it is suspended. The first time sqlite3rbu_vacuum() is called, to start +** an RBU vacuum operation, the state database should either not exist or +** be empty (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 5e8640a9f8..2d52262c71 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -20,8 +20,9 @@ #include #include -/* From main.c (apparently...) */ +/* From main.c */ extern const char *sqlite3ErrName(int); +extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx); @@ -66,7 +67,8 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ - {"bp_progress", 2, ""}, /* 5 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ {0,0,0} }; int iCmd; @@ -149,6 +151,22 @@ static int test_sqlite3rbu_cmd( break; } + case 6: /* db */ { + int bArg; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bArg) ){ + ret = TCL_ERROR; + }else{ + char zBuf[50]; + sqlite3 *db = sqlite3rbu_db(pRbu, bArg); + if( sqlite3TestMakePointerStr(interp, zBuf, (void*)db) ){ + ret = TCL_ERROR; + }else{ + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + } + } + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; @@ -187,6 +205,34 @@ static int test_sqlite3rbu( return TCL_OK; } +/* +** Tclcmd: sqlite3rbu_vacuum CMD +*/ +static int test_sqlite3rbu_vacuum( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3rbu *pRbu = 0; + const char *zCmd; + const char *zTarget; + const char *zStateDb = 0; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB STATE-DB"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + zStateDb = Tcl_GetString(objv[3]); + + pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + /* ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT */ @@ -274,6 +320,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3rbu", test_sqlite3rbu }, + { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum }, { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs }, { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs }, { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test }, diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test index 6560ea8755..be6c4568ce 100644 --- a/ext/session/sessionfault.test +++ b/ext/session/sessionfault.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] session_common.tcl] source $testdir/tester.tcl +ifcapable !session {finish_test; return} set testprefix sessionfault diff --git a/ext/session/sessionfault2.test b/ext/session/sessionfault2.test index 6b257cf41b..ffdc57b9bc 100644 --- a/ext/session/sessionfault2.test +++ b/ext/session/sessionfault2.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] session_common.tcl] source $testdir/tester.tcl +ifcapable !session {finish_test; return} set testprefix sessionfault2 do_execsql_test 1.0.0 { diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index b058d357e2..e333dffcc7 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -592,14 +592,19 @@ static int sessionChangeEqual( int iCol; /* Used to iterate through table columns */ for(iCol=0; iColnCol; iCol++){ - int n1 = sessionSerialLen(a1); - int n2 = sessionSerialLen(a2); + if( pTab->abPK[iCol] ){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); - if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ - return 0; + if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ + return 0; + } + a1 += n1; + a2 += n2; + }else{ + if( bLeftPkOnly==0 ) a1 += sessionSerialLen(a1); + if( bRightPkOnly==0 ) a2 += sessionSerialLen(a2); } - if( pTab->abPK[iCol] || bLeftPkOnly==0 ) a1 += n1; - if( pTab->abPK[iCol] || bRightPkOnly==0 ) a2 += n2; } return 1; diff --git a/manifest b/manifest index 9a3a2004c1..cc7dd90dc4 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2016-04-20T12:16:05.997 -F Makefile.in eba680121821b8a60940a81454316f47a341487a +C Merge\sthe\slatest\senhancements\sfrom\strunk. +D 2016-04-29T16:01:18.190 +F Makefile.in 9e816d0323e418fbc0f8b2c05fc14e0b3763d9e8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 @@ -222,7 +222,7 @@ F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 +F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 F ext/rbu/rbu1.test 42bd835e019eff789ec241017965277baeb658b1 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 @@ -243,12 +243,14 @@ F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 +F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 9097f1d95666dbef72ca61d5b6a13a84660735ac -F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2 -F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d +F ext/rbu/rbuvacuum.test 66e02cf299836770e718e95c36686be0b26dbda3 +F ext/rbu/sqlite3rbu.c bf36625990c6865ecf08bd844d8097ed2d0a6958 +F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930 +F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -289,9 +291,9 @@ F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596 -F ext/session/sessionfault.test d52cbb7bee48cc8ee80335e07eb72fcb6b15eb40 -F ext/session/sessionfault2.test ac1dfd77a0fb0ea310aee40a16645ef1b66d3f88 -F ext/session/sqlite3session.c beb300cd1b5c5054062c8e6e807b10475e363410 +F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 +F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 +F ext/session/sqlite3session.c beb43b6b888801bb006320bc236baa95f4cc32ae F ext/session/sqlite3session.h 64e9e7f185725ef43b97f4a9a0c0df0669844f1d F ext/session/test_session.c 464f2c8bf502795d95969387eb8e93f68c513c15 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 @@ -329,18 +331,18 @@ F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39 F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e -F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 +F src/expr.c 8796c0739b2ad091e6779253f62aad6e767e2be1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb -F src/func.c 552d300265aed09eea21f68ac742a440550c0062 +F src/func.c ef4c18c8a66143413ce41a58d582d2c14ddf78e1 F src/global.c c45ea22aff29334f6a9ec549235ac3357c970015 -F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 +F src/hash.c 55b5fb474100cee0b901edaf203e26c970940f36 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c e70f8f9e97624a232870ea5486e682c813ac3002 -F src/main.c 5ac9dccc03faadd6f867f67b9018ff41eeeadb46 +F src/loadext.c 8b3a73f0624c5f7cadbd5cb89940783bee1d39a6 +F src/main.c 405d13e3a4f7c5add9fb27702ae70ed0a6e32cca F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -359,38 +361,38 @@ F src/os.c 4d83917f072ad958fba9235136fa2ed43df47905 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f -F src/os_win.c b169437dff859e308b3726594094a2f8ca922941 +F src/os_unix.c 8422fba2eb592fbbb2d4006b6f2a67cad8951495 +F src/os_win.c 852fc2ff6084296348ed3739c548b2cf32df394e F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 476884ce8d8e845daf88f1b12b8ca446bac5fdc5 +F src/pager.c 41559a2b4461a6cf4b24bedb47d4e303d02729b5 F src/pager.h 6656436178c295d0668520739118e71a46163b8e F src/parse.y 2e1d371f99535ff29d1cd85052aa11cc636e21c8 -F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df -F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 -F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 +F src/pcache.c b3230ecfc7f797063fbe167f2845da363e8f07f8 +F src/pcache.h 6b865be765d1ebd06145219550b10921c7da7cc9 +F src/pcache1.c 7f51d2b541aab57596adf62db2c4bb025d34f04d F src/pragma.c faf42922bb7ab2f6672cb550356c1967abae3c84 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e -F src/select.c 30217121bdf6b587462150b8ee9e1467f7a6036b +F src/rowset.c 49eb91c588a2bab36647368e031dc5b66928149d +F src/select.c fd4a7ce2937497181063cfedb92058ac89491a5d F src/shell.c 14ff7f660530a52b117d110ba3390b7b2eb719b6 -F src/sqlite.h.in 64eb70a3b309751bebf73a5552a51244f68f0ea5 +F src/sqlite.h.in 9984129d86243424b765fcb3f147c697bd20bb54 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2 -F src/sqliteInt.h 0d9f22432cf553bab15e25cf168b57d93396922b +F src/sqliteInt.h e6ac0d8f8cbb062a57882858b1deed1a7924314b F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 9c4c4589d078de37813ded708d8838b338ffb060 -F src/test1.c 457c601302b8a0f5960dffd17b6a2877603841dd +F src/test1.c ee8fd5c32acb93fb6fe885ca4801cfab85f5563f F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b -F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f +F src/test3.c 0df6f8dbb4cbaa7106397c70a271fa6a43659042 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 -F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 +F src/test6.c 2c014d4977efd6107ec9eef3dfdec56ac516f824 F src/test7.c 9c89a4f1ed6bb13af0ed805b8d782bd83fcd57e3 F src/test8.c fa262391d3edea6490a71bfaa8fed477ccbbac75 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -402,14 +404,14 @@ F src/test_blob.c b2551a9b5573232db5f66f292307c37067937239 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f F src/test_config.c 5165f79b3aea5305627ac36dd7b9b5f124572da1 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 -F src/test_devsym.c 7e73009d5297b603c11c66d7c7d6723d4b5c73e1 +F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 F src/test_fs.c f10f840ca4f8c72e4837908bd8347ac4bcab074b F src/test_func.c 37453d346cfcf118774efd5bf6187f7e6a7e3254 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 -F src/test_journal.c da540964c675fde39487da2bc664096af97190d8 +F src/test_journal.c d3b83f2bcb7792c709e57abddc456a2b1818643a F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd F src/test_malloc.c 94c18e88d6df6d471a0d04ddb809f833d1739950 F src/test_multiplex.c eafc567ebe162e36f17b5062285dfe90461cf8e9 @@ -425,7 +427,7 @@ F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 0d138a8180a312bf996b37fa66da5c5799d4d57b F src/test_superlock.c 06797157176eb7085027d9dd278c0d7a105e3ec9 -F src/test_syscall.c 4889d374f5a5856b7951f7a67c6401f7b938c6f5 +F src/test_syscall.c 268c072541162564a882c57f54a6fee12ef4a4d2 F src/test_tclvar.c d86412527da65468ee6fa1b8607c65d0af736bc4 F src/test_thread.c af391ec03d23486dffbcc250b7e58e073f172af9 F src/test_vfs.c 4d02f38bfb8f7f273da7ba84bfe000f5babf206c @@ -439,9 +441,9 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 3e67ab3c0814635f355fb1f8ab010a2b9e016e7d F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 187a0a2aaa3c5d2ccd2ab0143b2fd9e86d6bc816 +F src/util.c 810ec3f22e2d1b62e66c30fe3621ebdedd23584d F src/vacuum.c c1a94413f8fa9d9b2cbe3af9a89377f777be676c -F src/vdbe.c 226c51cced9aa6f66748f626413d3f0348f2fcb8 +F src/vdbe.c 4f0378eb44406a6a61686dff7970fa314f0f44bd F src/vdbe.h 5591b5add447096e31288b5a0a78ec5d7b5c5170 F src/vdbeInt.h ddb157974436d87652de7dc641f7191496d9a8cd F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c @@ -510,7 +512,7 @@ F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d -F test/backup.test b79299a536a4c6d919094786595b95be56d02014 +F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 @@ -551,7 +553,7 @@ F test/capi3c.test 0b9edb0c2156a964b9271cd5ea7ae56736cc2fcb F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/cffault.test aadc1f61f8811cb600e3e069acbf8796f472a096 +F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 @@ -614,7 +616,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -646,7 +648,7 @@ F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 25385396082b67fd02ae0038b95a3b3575fe0519 -F test/e_vacuum.test 4d5b391384bb7d56bb9337d956f08035332421fc +F test/e_vacuum.test 120f29ea56bdce4d43279527ece894ab5d1729d3 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 @@ -661,7 +663,7 @@ F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 +F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/expr.test 79c3e7502d9e571553b85f0ecc8ff2ac7d0e4931 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 @@ -810,7 +812,7 @@ F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test d7b0a5f41123df1e43985b91b8b2e70f95282d21 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 40523db3aa76d62bda71c26f824fa0eabc420f0e +F test/hook.test 3b7b99d0eece6d279812c2aef6fa08bdfabc633e F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 @@ -827,7 +829,7 @@ F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 -F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac +F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 @@ -864,7 +866,7 @@ F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 8a5c0be6f0c260a5c7177c3b8f07c7856141038a F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 -F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 +F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d @@ -879,9 +881,9 @@ F test/like.test 81632c437a947bf1f7130b19537da6a1a844806a F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e -F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b +F test/loadext.test 42a3b8166dfcadcb0e0c8710dc520d97c31a8b98 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 +F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 @@ -970,7 +972,7 @@ F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test ae9ee0db5a30aecda9db8290ce3dd12e5f7bbaa1 +F test/pagerfault.test 42ff797b1e6426c141cc7ee8b7417c9f27427950 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 @@ -978,10 +980,10 @@ F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cd1fa041074ed08eeaa563e4d1bacb0c69337ec1 -F test/pragma.test dd5313eee9c6d9d4726593a68ede8768d3b50ccc +F test/permutations.test b6b3e165fdc1b8c82a820033646dbfc6a7a01746 +F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f -F test/pragma3.test 3f1984a04657331f838df5c519b443c2088df922 +F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb @@ -989,7 +991,7 @@ F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test 36cd78b178c4eb0401d4f25754ef410fbd9df2a7 +F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 @@ -1027,7 +1029,7 @@ F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test d926792a5e4d88fef0ddcddeb45d27ce75f7296c +F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328 F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0 F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61 @@ -1093,7 +1095,7 @@ F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a -F test/stat.test b65bad7120c52583b8f0054d99eff80718119a77 +F test/stat.test 66e95f97b9f724f9ab921d054ee0db3c2689f1ee F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f @@ -1103,7 +1105,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 -F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899 +F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906 @@ -1112,9 +1114,11 @@ F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 +F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 +F test/temptable2.test c3d8c138f493207612960bbd6a8c50e84975e2ee F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 +F test/tester.tcl 30c7a9be8601d1c1c9c93d013545ebcb28d64254 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1130,10 +1134,10 @@ F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b +F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 -F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac +F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d F test/tkt-385a5b56b9.test c0a06ada41d7f06b1686da0e718553f853771d1e F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 @@ -1144,7 +1148,7 @@ F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 F test/tkt-4ef7e3cfca.test 3965ae11cc9cf6e334f9d7d3c1e20bf8d56254b1 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 -F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa +F test/tkt-5d863f876e.test 726e76d725f6fe0eb2fc8a522b721b79807380ee F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 @@ -1160,7 +1164,7 @@ F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 -F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 +F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 @@ -1399,7 +1403,7 @@ F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e F test/wordcount.c 2a0a6c0d0e8e8bbbac1f06d72a6791828c37c0cf F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa -F test/zerodamage.test 2d725c214b883e25ae6bb85ef228ecdfa03c6a7b +F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 629d87562e0487c386db630033931d12d62e6372 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1415,7 +1419,7 @@ F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c cfbfe061a4b2766512f6b484882eee2c86a14506 +F tool/lemon.c 83318dff3911e47f2b85e136e56aa1c4674a2d2b F tool/lempar.c 404ea3dc27dbeed343f0e61b1d36e97b9f5f0fb6 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1485,7 +1489,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7381efff47c0a2f307478f196d03df7534f19e3 eba27d4d17a76884292667d570d542e580ee3e77 -R b2d94baa5aa70083709fce61087ef618 +P 1f709fbf931e4d75848fc90532e2bc67ccd47cd4 9d0a5ae00273686ea35b43bc2ffaa8775c176363 +R 53ca713e0bf5b0eed86d88e36153628f U drh -Z 09ace714760db63cbd8099bd8f9784df +Z e98bb20abec63c17e483c921950d1a1a diff --git a/manifest.uuid b/manifest.uuid index 82a5e61542..43d7e4a998 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f709fbf931e4d75848fc90532e2bc67ccd47cd4 \ No newline at end of file +91e5c07eaf884d3598df8eb54f0910a80df48397 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 094a454ab1..9e3b664e8d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2360,7 +2360,7 @@ static void cacheEntryClear(Parse *pParse, struct yColCache *p){ } p->iReg = 0; pParse->nColCache--; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); } @@ -2405,7 +2405,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ p->tempReg = 0; p->lru = pParse->iCacheCnt++; pParse->nColCache++; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); return; } } diff --git a/src/func.c b/src/func.c index 662a08f504..4feedc7440 100644 --- a/src/func.c +++ b/src/func.c @@ -1386,6 +1386,14 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ diff --git a/src/hash.c b/src/hash.c index b5886e0641..eea2dd1ac2 100644 --- a/src/hash.c +++ b/src/hash.c @@ -55,7 +55,7 @@ void sqlite3HashClear(Hash *pH){ static unsigned int strHash(const char *z){ unsigned int h = 0; unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; } return h; @@ -148,7 +148,7 @@ static HashElem *findElementWithHash( int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - if( pH->ht ){ + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; h = strHash(pKey) % pH->htsize; pEntry = &pH->ht[h]; diff --git a/src/loadext.c b/src/loadext.c index 495001e55a..f881d99710 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -464,8 +464,9 @@ static int sqlite3LoadExtension( /* Ticket #1863. To avoid a creating security problems for older ** applications that relink against newer versions of SQLite, the ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. */ if( (db->flags & SQLITE_LoadExtension)==0 ){ if( pzErrMsg ){ @@ -604,9 +605,9 @@ void sqlite3CloseExtensions(sqlite3 *db){ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension; + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~SQLITE_LoadExtension; + db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; diff --git a/src/main.c b/src/main.c index 9f773667af..30370f8cab 100644 --- a/src/main.c +++ b/src/main.c @@ -804,6 +804,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/os_unix.c b/src/os_unix.c index aa86f00d70..01de00e0dd 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4288,10 +4288,12 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } } if( pInode->bProcessLock==0 ){ @@ -5416,14 +5418,14 @@ static const char *unixTempFileDir(void){ if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - for(i=0; ilocktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, @@ -3765,10 +3764,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } } rc = winOpen(pDbFd->pVfs, diff --git a/src/pager.c b/src/pager.c index 363ddf7f33..36a936a9c3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -875,6 +875,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -963,7 +964,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -1175,6 +1176,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -1868,13 +1871,17 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ - assert( !MEMDB ); - pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; } pPager->journalOff = 0; @@ -1918,6 +1925,25 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on the pager passed as the only argument is +** being committed. This function returns true if all dirty pages should +** be flushed to disk, or false otherwise. Pages should be flushed to disk +** unless one of the following is true: +** +** * The db is an in-memory database. +** +** * The db is a temporary database and the db file has not been opened. +** +** * The db is a temporary database and the cache contains less than +** C/4 dirty pages, where C is the configured cache-size. +*/ +static int pagerFlushOnCommit(Pager *pPager){ + if( pPager->tempFile==0 ) return 1; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -2055,7 +2081,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ pagerFreeBitvecs(pPager); pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); + if( MEMDB || pagerFlushOnCommit(pPager) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ @@ -2340,7 +2370,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -2423,9 +2453,13 @@ static int pager_playback_one_page( ** be written out into the database file before its journal file ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. + ** + ** Update: Another exception is for temp files that are not + ** in-memory databases. In this case the page may have been dirty + ** at the start of the transaction. */ assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); + if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg); } pager_set_pagehash(pPg); @@ -3227,6 +3261,8 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the @@ -3234,14 +3270,11 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -4317,8 +4350,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -4992,6 +5026,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -5084,17 +5119,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -5180,7 +5215,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -5276,7 +5311,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } @@ -5419,7 +5454,7 @@ int sqlite3PagerGet( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ @@ -5486,7 +5521,8 @@ int sqlite3PagerGet( goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -5628,24 +5664,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -6019,6 +6055,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -6325,17 +6362,21 @@ int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( (rc = sqlite3FaultSim(400))!=SQLITE_OK ) return rc; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -6674,10 +6715,10 @@ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* @@ -6957,7 +6998,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -7014,7 +7055,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -7031,7 +7072,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ + if( pPager->tempFile ){ assert( pPgOld ); sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); @@ -7284,7 +7325,8 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif @@ -7463,6 +7505,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; diff --git a/src/pcache.c b/src/pcache.c index 61e7587422..f700c2ff66 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -254,7 +254,7 @@ sqlite3_pcache_page *sqlite3PcacheFetch( /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because new clean pages are available for reuse and the cache +** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -439,6 +439,17 @@ void sqlite3PcacheCleanAll(PCache *pCache){ } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -484,7 +495,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } @@ -675,6 +686,17 @@ void sqlite3PcacheShrink(PCache *pCache){ */ int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; +} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* diff --git a/src/pcache.h b/src/pcache.h index 475c04c061..04bce289d0 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -99,6 +99,7 @@ void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ void sqlite3PcacheMove(PgHdr*, Pgno); @@ -173,4 +174,7 @@ void sqlite3PCacheSetDefault(void); int sqlite3HeaderSizePcache(void); int sqlite3HeaderSizePcache1(void); +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index d168e7fbcb..5fe963ad05 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -690,8 +690,8 @@ static int pcache1Init(void *NotUsed){ #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); } #endif if( pcache1.separateCache diff --git a/src/rowset.c b/src/rowset.c index c2e73ed72e..a70264ed1b 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -57,8 +57,9 @@ ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. */ #include "sqliteInt.h" @@ -179,7 +180,9 @@ void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ @@ -213,7 +216,9 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -335,23 +340,29 @@ static struct RowSetEntry *rowSetNDeepTree( ){ struct RowSetEntry *p; /* Root of the new tree */ struct RowSetEntry *pLeft; /* Left subtree */ - if( *ppList==0 ){ - return 0; + if( *ppList==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Prevent unnecessary deep recursion when we run out of entries */ + return 0; } - if( iDepth==1 ){ + if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; - return p; } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } @@ -378,59 +389,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } -/* -** Take all the entries on p->pEntry and on the trees in p->pForest and -** sort them all together into one big ordered list on p->pEntry. -** -** This routine should only be called once in the life of a RowSet. -*/ -static void rowSetToList(RowSet *p){ - - /* This routine is called only once */ - assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); - - if( (p->rsFlags & ROWSET_SORTED)==0 ){ - p->pEntry = rowSetEntrySort(p->pEntry); - } - - /* While this module could theoretically support it, sqlite3RowSetNext() - ** is never called after sqlite3RowSetText() for the same RowSet. So - ** there is never a forest to deal with. Should this change, simply - ** remove the assert() and the #if 0. */ - assert( p->pForest==0 ); -#if 0 - while( p->pForest ){ - struct RowSetEntry *pTree = p->pForest->pLeft; - if( pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(pTree, &pHead, &pTail); - p->pEntry = rowSetEntryMerge(p->pEntry, pHead); - } - p->pForest = p->pForest->pRight; - } -#endif - p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ -} - /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ sqlite3RowSetClear(p); } return 1; @@ -453,13 +442,15 @@ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. */ - if( iBatch!=pRowSet->iBatch ){ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entiries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ diff --git a/src/select.c b/src/select.c index b86e040f0f..eb37536b6f 100644 --- a/src/select.c +++ b/src/select.c @@ -3785,14 +3785,18 @@ static int pushDownWhereTerms( ){ Expr *pNew; int nChng = 0; + Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; - if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pSubq->selFlags & SF_Aggregate ); - testcase( pSubq->selFlags & SF_Recursive ); - return 0; /* restrictions (1) and (2) */ + for(pX=pSubq; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + testcase( pX->selFlags & SF_Aggregate ); + testcase( pX->selFlags & SF_Recursive ); + testcase( pX!=pSubq ); + return 0; /* restrictions (1) and (2) */ + } } if( pSubq->pLimit!=0 ){ - return 0; /* restriction (3) */ + return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d50d826b41..795236f587 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1932,12 +1932,30 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back. ** +**
SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
+**
^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argment to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
+** ** */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ #define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ /* @@ -5474,9 +5492,18 @@ int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, ** otherwise an error will be returned. ** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** ** See also the [load_extension() SQL function]. */ int sqlite3_load_extension( @@ -5499,6 +5526,17 @@ int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API. +** +** Security warning: It is recommended that extension loading +** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 889e0b2229..24f8e9d300 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -15,6 +15,33 @@ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ +/* Special Comments: +** +** Some comments have special meaning to the tools that measure test +** coverage: +** +** NO_TEST - The branches on this line are not +** measured by branch coverage. This is +** used on lines of code that actually +** implement parts of coverage testing. +** +** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread +** that would be harmless and undetectable +** if it did occur. +** +** In all cases, the special comment must be enclosed in the usual +** slash-asterisk...asterisk-slash comment marks, with no spaces between the +** asterisks and the comment text. +*/ + /* ** Make sure that rand_s() is available on Windows systems with MSVC 2005 ** or higher. @@ -1383,13 +1410,14 @@ struct sqlite3 { #define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ #define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ -#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ -#define SQLITE_CellSizeCk 0x10000000 /* Check btree cell sizes on load */ -#define SQLITE_Fts3Tokenizer 0x20000000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_LoadExtFunc 0x00800000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x01000000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x02000000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x04000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x08000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_Vacuum 0x10000000 /* Currently in a VACUUM */ +#define SQLITE_CellSizeCk 0x20000000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x40000000 /* Enable fts3_tokenizer(2) */ /* diff --git a/src/test1.c b/src/test1.c index 0c5af822c9..5b7581af7b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1271,7 +1271,7 @@ static int sqlite3_mprintf_int64( return TCL_ERROR; } for(i=2; i<5; i++){ - if( sqlite3Atoi64(argv[i], &a[i-2], 1000000, SQLITE_UTF8) ){ + if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){ Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0); return TCL_ERROR; } @@ -5213,7 +5213,9 @@ static int vfs_unregister_all( /* ** tclcmd: vfs_reregister_all ** -** Restore all VFSes that were removed using vfs_unregister_all +** Restore all VFSes that were removed using vfs_unregister_all. Taking +** care to put the linked list back together in the same order as it was +** in before vfs_unregister_all was invoked. */ static int vfs_reregister_all( ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ @@ -5222,8 +5224,8 @@ static int vfs_reregister_all( Tcl_Obj *CONST objv[] /* Command arguments */ ){ int i; - for(i=0; i=0; i--){ + sqlite3_vfs_register(apVfs[i], 1); } return TCL_OK; } @@ -6988,6 +6990,7 @@ static int test_sqlite3_db_config( { "FKEY", SQLITE_DBCONFIG_ENABLE_FKEY }, { "TRIGGER", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "FTS3_TOKENIZER", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "LOAD_EXTENSION", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, }; int i; int v; diff --git a/src/test3.c b/src/test3.c index 2a41068e5f..817e8a88fd 100644 --- a/src/test3.c +++ b/src/test3.c @@ -547,7 +547,7 @@ static int btree_from_db( /* ** Usage: btree_ismemdb ID ** -** Return true if the B-Tree is in-memory. +** Return true if the B-Tree is currently stored entirely in memory. */ static int btree_ismemdb( void *NotUsed, @@ -557,6 +557,7 @@ static int btree_ismemdb( ){ Btree *pBt; int res; + sqlite3_file *pFile; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], @@ -566,7 +567,8 @@ static int btree_ismemdb( pBt = sqlite3TestTextToPtr(argv[1]); sqlite3_mutex_enter(pBt->db->mutex); sqlite3BtreeEnter(pBt); - res = sqlite3PagerIsMemdb(sqlite3BtreePager(pBt)); + pFile = sqlite3PagerFile(sqlite3BtreePager(pBt)); + res = (pFile->pMethods==0); sqlite3BtreeLeave(pBt); sqlite3_mutex_leave(pBt->db->mutex); Tcl_SetObjResult(interp, Tcl_NewBooleanObj(res)); diff --git a/src/test6.c b/src/test6.c index 306482dcd3..2a09122c6c 100644 --- a/src/test6.c +++ b/src/test6.c @@ -701,6 +701,10 @@ static int cfCurrentTime(sqlite3_vfs *pCfVfs, double *pTimeOut){ sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData; return pVfs->xCurrentTime(pVfs, pTimeOut); } +static int cfGetLastError(sqlite3_vfs *pCfVfs, int n, char *z){ + sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData; + return pVfs->xGetLastError(pVfs, n, z); +} static int processDevSymArgs( Tcl_Interp *interp, @@ -827,7 +831,7 @@ static int crashEnableCmd( cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ cfCurrentTime, /* xCurrentTime */ - 0, /* xGetlastError */ + cfGetLastError, /* xGetLastError */ 0, /* xCurrentTimeInt64 */ }; @@ -940,6 +944,27 @@ static int devSymObjCmd( devsym_register(iDc, iSectorSize); return TCL_OK; + +} + +/* +** tclcmd: unregister_devsim +*/ +static int dsUnregisterObjCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void devsym_unregister(void); + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + devsym_unregister(); + return TCL_OK; } /* @@ -1010,6 +1035,7 @@ int Sqlitetest6_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0); + Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "unregister_jt_vfs", jtUnregisterObjCmd, 0, 0); #endif diff --git a/src/test_devsym.c b/src/test_devsym.c index 5fd0935846..9a1ba09d69 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -396,4 +396,11 @@ void devsym_register(int iDeviceChar, int iSectorSize){ } } +void devsym_unregister(){ + sqlite3_vfs_unregister(&devsym_vfs); + g.pVfs = 0; + g.iDeviceChar = 0; + g.iSectorSize = 0; +} + #endif diff --git a/src/test_journal.c b/src/test_journal.c index 84c80546a6..4e63bccf76 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -160,6 +160,7 @@ static int jtRandomness(sqlite3_vfs*, int nByte, char *zOut); static int jtSleep(sqlite3_vfs*, int microseconds); static int jtCurrentTime(sqlite3_vfs*, double*); static int jtCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int jtGetLastError(sqlite3_vfs*, int, char*); static sqlite3_vfs jt_vfs = { 2, /* iVersion */ @@ -179,7 +180,7 @@ static sqlite3_vfs jt_vfs = { jtRandomness, /* xRandomness */ jtSleep, /* xSleep */ jtCurrentTime, /* xCurrentTime */ - 0, /* xGetLastError */ + jtGetLastError, /* xGetLastError */ jtCurrentTimeInt64 /* xCurrentTimeInt64 */ }; @@ -285,9 +286,10 @@ static int jtRead( ** b) The file-name specified when the file was opened matches ** all but the final 8 characters of the journal file name. ** -** c) There is currently a reserved lock on the file. +** c) There is currently a reserved lock on the file. This +** condition is waived if the noLock argument is non-zero. **/ -static jt_file *locateDatabaseHandle(const char *zJournal){ +static jt_file *locateDatabaseHandle(const char *zJournal, int noLock){ jt_file *pMain = 0; enterJtMutex(); for(pMain=g.pList; pMain; pMain=pMain->pNext){ @@ -295,7 +297,7 @@ static jt_file *locateDatabaseHandle(const char *zJournal){ if( (pMain->flags&SQLITE_OPEN_MAIN_DB) && ((int)strlen(pMain->zName)==nName) && 0==memcmp(pMain->zName, zJournal, nName) - && (pMain->eLock>=SQLITE_LOCK_RESERVED) + && ((pMain->eLock>=SQLITE_LOCK_RESERVED) || noLock) ){ break; } @@ -517,7 +519,7 @@ static int jtWrite( jt_file *p = (jt_file *)pFile; if( p->flags&SQLITE_OPEN_MAIN_JOURNAL ){ if( iOfst==0 ){ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); assert( pMain ); if( iAmt==28 ){ @@ -562,7 +564,7 @@ static int jtWrite( rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); if( (p->flags&SQLITE_OPEN_MAIN_JOURNAL) && iAmt==12 ){ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); int rc2 = readJournalFile(p, pMain); if( rc==SQLITE_OK ) rc = rc2; } @@ -576,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){ jt_file *p = (jt_file *)pFile; if( p->flags&SQLITE_OPEN_MAIN_JOURNAL && size==0 ){ /* Truncating a journal file. This is the end of a transaction. */ - jt_file *pMain = locateDatabaseHandle(p->zName); + jt_file *pMain = locateDatabaseHandle(p->zName, 0); closeTransaction(pMain); } if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){ @@ -604,11 +606,10 @@ static int jtSync(sqlite3_file *pFile, int flags){ ** jt_file.pWritable bitvec of the main database file associated with ** this journal file. */ - pMain = locateDatabaseHandle(p->zName); - assert(pMain); + pMain = locateDatabaseHandle(p->zName, 0); /* Set the bitvec values */ - if( pMain->pWritable ){ + if( pMain && pMain->pWritable ){ pMain->nSync++; rc = readJournalFile(p, pMain); if( rc!=SQLITE_OK ){ @@ -730,7 +731,7 @@ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ int nPath = (int)strlen(zPath); if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){ /* Deleting a journal file. The end of a transaction. */ - jt_file *pMain = locateDatabaseHandle(zPath); + jt_file *pMain = locateDatabaseHandle(zPath, 0); if( pMain ){ closeTransaction(pMain); } @@ -825,6 +826,10 @@ static int jtCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ return g.pVfs->xCurrentTimeInt64(g.pVfs, pTimeOut); } +static int jtGetLastError(sqlite3_vfs *pVfs, int n, char *z){ + return g.pVfs->xGetLastError(g.pVfs, n, z); +} + /************************************************************************** ** Start of public API. */ diff --git a/src/test_syscall.c b/src/test_syscall.c index f9abc1e46d..f1d5c61bc3 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -722,14 +722,20 @@ static int test_syscall( }; int iCmd; int rc; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); if( objc<2 ){ Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); return TCL_ERROR; } - rc = Tcl_GetIndexFromObjStruct(interp, - objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd - ); + if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){ + Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0); + rc = TCL_ERROR; + }else{ + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd + ); + } if( rc!=TCL_OK ) return rc; return aCmd[iCmd].xCmd(clientData, interp, objc, objv); } diff --git a/src/util.c b/src/util.c index 2f77a6033f..db6163c3c7 100644 --- a/src/util.c +++ b/src/util.c @@ -355,7 +355,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -368,7 +368,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i=zEnd ) goto do_atof_calc; @@ -416,7 +414,12 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; - if( z>=zEnd ) goto do_atof_calc; + + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer + ** is obtained even if the branch is omitted */ + if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + /* get sign of exponent */ if( *z=='-' ){ esign = -1; @@ -433,9 +436,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ } /* skip trailing spaces */ - if( nDigits && eValid ){ - while( z0 ){ - while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10; - }else{ - while( !(s%10) && e>0 ) e--,s/=10; + /* Attempt to reduce exponent. + ** + ** Branches that are not required for the correct answer but which only + ** help to obtain the correct answer faster are marked with special + ** comments, as a hint to the mutation tester. + */ + while( e>0 ){ /*OPTIMIZATION-IF-TRUE*/ + if( esign>0 ){ + if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ + s *= 10; + }else{ + if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ + s /= 10; + } + e--; } /* adjust the sign of significand */ s = sign<0 ? -s : s; - /* if exponent, scale significand as appropriate - ** and store in result. */ - if( e ){ + if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ + result = (double)s; + }else{ LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ - if( e>307 && e<342 ){ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else if( e>=342 ){ - if( esign<0 ){ - result = 0.0*s; - }else{ - result = 1e308*1e308*s; /* Infinity */ + if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ + if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ + while( e%308 ) { scale *= 1.0e+1; e -= 1; } + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else{ assert( e>=342 ); + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } } }else{ /* 1.0e+22 is the largest power of 10 than can be @@ -494,8 +505,6 @@ do_atof_calc: result = s * scale; } } - } else { - result = (double)s; } } @@ -503,7 +512,7 @@ do_atof_calc: *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z>=zEnd && nDigits>0 && eValid && nonNum==0; + return z==zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -565,7 +574,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ int neg = 0; /* assume positive */ int i; int c = 0; - int nonNum = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ const char *zStart; const char *zEnd = zNum + length; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); @@ -576,7 +585,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i19*incr || nonNum ){ + if( &zNum[i]19*incr /* Too many digits */ + || nonNum /* UTF16 with high-order bytes non-zero */ + ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; @@ -646,7 +658,6 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') - && sqlite3Isxdigit(z[2]) ){ u64 u = 0; int i, k; @@ -1408,7 +1419,7 @@ LogEst sqlite3LogEst(u64 x){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ - while( x>255 ){ y += 40; x >>= 4; } + while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ while( x>15 ){ y += 10; x >>= 1; } } return a[x&7] + y - 10; diff --git a/src/vdbe.c b/src/vdbe.c index ab66ada440..ccac41c5b1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -215,7 +215,7 @@ static VdbeCursor *allocateCursor( (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); assert( iCur>=0 && iCurnCursor ); - if( p->apCsr[iCur] ){ + if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } @@ -292,7 +292,7 @@ static void applyAffinity( if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - if( (pRec->flags & MEM_Int)==0 ){ + if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ if( (pRec->flags & MEM_Real)==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); }else{ @@ -302,10 +302,13 @@ static void applyAffinity( }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real ** representation (blob and NULL do not get converted) but no string - ** representation. - */ - if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc, 1); + ** representation. It would be harmless to repeat the conversion if + ** there is already a string rep, but it is pointless to waste those + ** CPU cycles. */ + if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc, 1); + } } pRec->flags &= ~(MEM_Real|MEM_Int); } @@ -542,7 +545,7 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ){ + if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/ return out2PrereleaseWithClear(pOut); }else{ pOut->flags = MEM_Int; diff --git a/test/backup.test b/test/backup.test index 3b1e1db9e5..829b32452d 100644 --- a/test/backup.test +++ b/test/backup.test @@ -164,7 +164,7 @@ foreach zOpenScript [list { set file_dest temp }] { foreach rows_dest {0 3 10} { -foreach pgsz_dest {512 1024 2048} { +foreach pgsz_dest {512 1024 2048 4096} { foreach nPagePerStep {1 200} { # Open the databases. @@ -176,17 +176,16 @@ foreach nPagePerStep {1 200} { # in-memory destination is only possible if the initial destination # page size is the same as the source page size (in this case 1024 bytes). # - set isMemDest [expr { - $zDestFile eq ":memory:" || $file_dest eq "temp" && $TEMP_STORE>=2 - }] + set isMemDest [expr { $zDestFile eq ":memory:" || $file_dest eq "temp" }] - if { $isMemDest==0 || $pgsz_dest == 1024 } { - if 0 { - puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" - puts -nonewline " (as $db_dest.$file_dest)" - puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" - puts "" - } + if 0 { + puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile" + puts -nonewline " (as $db_dest.$file_dest)" + puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest" + puts "" + } + + if { $isMemDest==0 || $pgsz_dest==1024 || $rows_dest==0 } { # Set up the content of the source database. execsql { diff --git a/test/cffault.test b/test/cffault.test index 79cefd24c4..0d029ece37 100644 --- a/test/cffault.test +++ b/test/cffault.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix cacheflush +set testprefix cffault source $testdir/malloc_common.tcl # Run the supplied SQL on a copy of the database currently stored on diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 2541a1a823..eff4b0207f 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -86,7 +86,7 @@ do_test 2.3 { db_write db 1 } {0 4 0} do_test 2.4 { db_write db 0 } {0 0 0} do_test 2.5 { db_write db 1 } {0 0 0} -ifcapable wal { +if {[wal_is_capable]} { do_test 2.6 { execsql { PRAGMA journal_mode = WAL } db_write db 1 diff --git a/test/e_vacuum.test b/test/e_vacuum.test index 1113a1fb6e..c3e517a892 100644 --- a/test/e_vacuum.test +++ b/test/e_vacuum.test @@ -176,7 +176,7 @@ if {![nonzero_reserved_bytes]} { # EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the # auto_vacuum support property can be changed using VACUUM. # - ifcapable wal { + if {[wal_is_capable]} { do_test e_vacuum-1.3.3.1 { execsql { PRAGMA journal_mode = wal } execsql { PRAGMA page_size ; PRAGMA auto_vacuum } diff --git a/test/exists.test b/test/exists.test index fb73797d29..5f9e36eb00 100644 --- a/test/exists.test +++ b/test/exists.test @@ -19,6 +19,7 @@ source $testdir/lock_common.tcl foreach jm {rollback wal} { + if {![wal_is_capable] && $jm=="wal"} continue set testprefix exists-$jm diff --git a/test/hook.test b/test/hook.test index 45e2959591..12e5bc0656 100644 --- a/test/hook.test +++ b/test/hook.test @@ -704,7 +704,7 @@ do_execsql_test 7.5.2.0 { ALTER TABLE t8 ADD COLUMN c DEFAULT 'xxx'; } -ifcapable !session { +if 0 { # At time of writing, these two are broken. They demonstrate that the # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE # has been used to add a column with a default value other than NULL. diff --git a/test/incrvacuum2.test b/test/incrvacuum2.test index 2219d54195..6ec66c9deb 100644 --- a/test/incrvacuum2.test +++ b/test/incrvacuum2.test @@ -134,7 +134,7 @@ do_test incrvacuum2-3.2 { integrity_check incrvacuum2-3.3 -ifcapable wal { +if {[wal_is_capable]} { # At one point, when a specific page was being extracted from the b-tree # free-list (e.g. during an incremental-vacuum), all trunk pages that # occurred before the specific page in the free-list trunk were being diff --git a/test/journal2.test b/test/journal2.test index 8f9b4d0b71..7cbe0ef2c5 100644 --- a/test/journal2.test +++ b/test/journal2.test @@ -204,7 +204,7 @@ db close # delete the journal file when committing the transaction that switches # the system to WAL mode. # -ifcapable wal { +if {[wal_is_capable]} { do_test journal2-2.1 { faultsim_delete_and_reopen set ::oplog [list] diff --git a/test/loadext.test b/test/loadext.test index 7ba4c0cf77..e6ba21e187 100644 --- a/test/loadext.test +++ b/test/loadext.test @@ -111,7 +111,7 @@ do_test loadext-1.2 { # do_test loadext-1.3 { sqlite3 db2 test.db - sqlite3_enable_load_extension db2 1 + sqlite3_db_config db2 SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1 catchsql { SELECT half(1.0); } db2 @@ -256,6 +256,7 @@ do_test loadext-4.2 { } } {0 {{}}} +# disable all extension loading do_test loadext-4.3 { sqlite3_enable_load_extension db 0 catchsql { @@ -263,6 +264,15 @@ do_test loadext-4.3 { } } {1 {not authorized}} +# enable C-api extension loading only. Show that the SQL function +# still does not work. +do_test loadext-4.4 { + sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1 + catchsql { + SELECT load_extension($::testextension,'testloadext_init') + } +} {1 {not authorized}} + source $testdir/malloc_common.tcl diff --git a/test/lock.test b/test/lock.test index f785c4bb23..534aa3b9a4 100644 --- a/test/lock.test +++ b/test/lock.test @@ -423,8 +423,9 @@ do_test lock-6.5 { # * there exists one or more active read-only statements, and # * a transaction that modified zero database pages is committed. # -set temp_status unlocked -if {$TEMP_STORE>=2} {set temp_status unknown} +#set temp_status unlocked +#if {$TEMP_STORE>=2} {set temp_status unknown} +set temp_status unknown do_test lock-7.1 { set STMT [sqlite3_prepare $DB "SELECT * FROM sqlite_master" -1 TAIL] sqlite3_step $STMT diff --git a/test/pagerfault.test b/test/pagerfault.test index c0f5de69ac..2e70171b20 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -684,22 +684,24 @@ do_faultsim_test pagerfault-14a -prep { # is not possible to change the page-size of an in-memory database. Even # using the backup API. # -if {$TEMP_STORE<2} { - do_faultsim_test pagerfault-14b -prep { - catch { db2 close } - faultsim_restore_and_reopen +# Update: It is no longer possible to change the page size of any temp +# database after it has been created. +# +do_faultsim_test pagerfault-14b -prep { + catch { db2 close } + faultsim_restore_and_reopen sqlite3 db2 "" db2 eval { PRAGMA page_size = 4096; CREATE TABLE xx(a) } - } -body { - sqlite3_backup B db2 main db main - B step 200 - set rc [B finish] - if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} - if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } - set {} {} - } -test { - faultsim_test_result {0 {}} {1 {sqlite3_backup_init() failed}} - } +} -body { + sqlite3_backup B db2 main db main + B step 200 + set rc [B finish] + if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR} + if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] } + set {} {} +} -test { + faultsim_test_result {1 {attempt to write a readonly database}} \ + {1 {sqlite3_backup_init() failed}} } do_faultsim_test pagerfault-14c -prep { diff --git a/test/permutations.test b/test/permutations.test index 308d8bde77..df3f500cda 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -728,7 +728,7 @@ test_suite "inmemory_journal" -description { ioerr.test ioerr2.test ioerr3.test ioerr4.test ioerr5.test vacuum3.test incrblob_err.test diskfull.test backup_ioerr.test e_fts3.test fts3cov.test fts3malloc.test fts3rnd.test - fts3snippet.test mmapfault.test + fts3snippet.test mmapfault.test sessionfault.test sessionfault2.test # Exclude test scripts that use tcl IO to access journal files or count # the number of fsync() calls. @@ -942,7 +942,9 @@ test_suite "journaltest" -description { unregister_jt_vfs } -files [test_set $::allquicktests -exclude { wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test - async4.test bigfile.test backcompat.test + async4.test bigfile.test backcompat.test e_wal* fstat.test mmap2.test + pager1.test syscall.test tkt3457.test *malloc* mmap* multiplex* nolock* + pager2.test *fault* rowal* snapshot* superlock* symlink.test }] if {[info commands register_demovfs] != ""} { @@ -1050,15 +1052,13 @@ proc run_tests {name args} { set ::G(isquick) 1 set ::G(perm:dbconfig) $options(-dbconfig) - uplevel $options(-initialize) - foreach file [lsort $options(-files)] { + uplevel $options(-initialize) if {[file tail $file] == $file} { set file [file join $::testdir $file] } slave_test_file $file + uplevel $options(-shutdown) } - uplevel $options(-shutdown) - unset ::G(perm:name) unset ::G(perm:prefix) unset ::G(perm:presql) diff --git a/test/pragma.test b/test/pragma.test index e3f6f8cb0f..44498d076b 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1083,7 +1083,20 @@ do_test pragma-8.2.15 { # "memory" or "disk" as appropriate. # proc check_temp_store {} { - db eval {CREATE TEMP TABLE IF NOT EXISTS a(b)} + db eval { + PRAGMA temp.cache_size = 1; + CREATE TEMP TABLE IF NOT EXISTS a(b); + DELETE FROM a; + INSERT INTO a VALUES(randomblob(1000)); + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + INSERT INTO a SELECT * FROM a; + } db eval {PRAGMA database_list} { if {$name=="temp"} { set bt [btree_from_db db 1] diff --git a/test/pragma3.test b/test/pragma3.test index 1a67d72350..8a4f4f9d84 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -221,7 +221,7 @@ ifcapable shared_cache { # This will not work with the in-memory journal permutation, as opening # [db2] switches the journal mode back to "memory" # -ifcapable wal { +if {[wal_is_capable]} { if {[permutation]!="inmemory_journal"} { sqlite3 db test.db diff --git a/test/quota.test b/test/quota.test index 7af55a8330..5d0bda3ad3 100644 --- a/test/quota.test +++ b/test/quota.test @@ -527,4 +527,6 @@ do_faultsim_test quota-5.6 -prep { } catch { sqlite3_quota_shutdown } +catch { db close } +forcedelete test.db finish_test diff --git a/test/select4.test b/test/select4.test index 1f29f29073..51a1b1c4c5 100644 --- a/test/select4.test +++ b/test/select4.test @@ -968,6 +968,42 @@ do_execsql_test select4-16.3 { ORDER BY t3.a; } {95 96 97 98 99} +# Ticket https://www.sqlite.org/src/tktview/f7f8c97e975978d45 on 2016-04-25 +# +# The where push-down optimization from 2015-06-02 is suppose to disable +# on aggregate subqueries. But if the subquery is a compound where the +# last SELECT is non-aggregate but some other SELECT is an aggregate, the +# test is incomplete and the optimization is not properly disabled. +# +# The following test cases verify that the fix works. +# +do_execsql_test select4-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a int, b int); + INSERT INTO t1 VALUES(1,2),(1,18),(2,19); + SELECT x, y FROM ( + SELECT 98 AS x, 99 AS y + UNION + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 20 98 99} +do_execsql_test select4-17.2 { + SELECT x, y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a + UNION + SELECT 98 AS x, 99 AS y + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 20 98 99} +do_catchsql_test select4-17.3 { + SELECT x, y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a LIMIT 3 + UNION + SELECT 98 AS x, 99 AS y + ) AS w WHERE y>=20 + ORDER BY +x; +} {1 {LIMIT clause should come after UNION not before}} diff --git a/test/stat.test b/test/stat.test index 95586e94bc..d106894d71 100644 --- a/test/stat.test +++ b/test/stat.test @@ -37,7 +37,7 @@ do_execsql_test stat-0.0 { SELECT * FROM stat; } {} -ifcapable wal { +if {[wal_is_capable]} { do_execsql_test stat-0.1 { PRAGMA journal_mode = WAL; PRAGMA journal_mode = delete; diff --git a/test/sync.test b/test/sync.test index 2ee636a266..210039acb5 100644 --- a/test/sync.test +++ b/test/sync.test @@ -81,6 +81,7 @@ do_test sync-1.3 { set sqlite_sync_count } 11 ifcapable pager_pragmas { +if {[permutation]!="journaltest"} { do_test sync-1.4 { set sqlite_sync_count 0 execsql { @@ -94,6 +95,7 @@ ifcapable pager_pragmas { set sqlite_sync_count } 0 } +} finish_test diff --git a/test/tempfault.test b/test/tempfault.test new file mode 100644 index 0000000000..4eae116a72 --- /dev/null +++ b/test/tempfault.test @@ -0,0 +1,134 @@ +# 2016 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for fault-injection when SQLite is used with +# a temp file database. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix tempfault + +# sqlite3_memdebug_vfs_oom_test 0 + +do_faultsim_test 1 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } +} -body { + execsql { INSERT INTO t1 VALUES(5, 6) } +} -test { + faultsim_test_result {0 {}} + set rc [catch { execsql { SELECT * FROM t1 } } msg] + if {$rc==0 && $msg != "1 2 3 4 5 6" && $msg != "1 2 3 4"} { + error "data mismatch 1: $msg" + } + if {$testrc==0 && $msg != "1 2 3 4 5 6"} { + error "data mismatch 2: $msg" + } + faultsim_integrity_check +} + +do_faultsim_test 2 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +catch { db close } +do_faultsim_test 2.1 -faults * -prep { + if {[info commands db]==""} { + sqlite3 db "" + execsql { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } + } +} -body { + execsql { UPDATE t1 SET a = randomblob(99) } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 3 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} + faultsim_integrity_check db +} + +do_faultsim_test 4 -faults * -prep { + sqlite3 db "" + db eval { + PRAGMA page_size = 1024; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b, a); + WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET a = randomblob(99); + SAVEPOINT abc; + UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0; + ROLLBACK TO abc; + UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0; + ROLLBACK TO abc; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} +} + +sqlite3_memdebug_vfs_oom_test 1 +finish_test diff --git a/test/temptable2.test b/test/temptable2.test new file mode 100644 index 0000000000..890b3b996d --- /dev/null +++ b/test/temptable2.test @@ -0,0 +1,356 @@ +# 2016 March 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix temptable2 + +do_execsql_test 1.1 { + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); +} + +do_execsql_test 1.2 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X; +} {} + +do_execsql_test 1.3 { + PRAGMA temp.integrity_check; +} {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.1 { + CREATE TEMP TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 2); +} {} + +do_execsql_test 2.2 { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + SELECT * FROM t2; +} {1 2 3 4} + +do_execsql_test 2.3 { + ROLLBACK; + SELECT * FROM t2; +} {1 2} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + ROLLBACK; +} +do_execsql_test 3.1.3 { + SELECT count(*) FROM t1; +} {1000} +do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok} + +do_execsql_test 3.2.1 { + BEGIN; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; + SAVEPOINT abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1; + ROLLBACK TO abc; + UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2; + COMMIT; +} +do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t1; + PRAGMA temp.page_count; +} {10 9} + +do_execsql_test 4.1.2 { + BEGIN; + UPDATE t1 SET b=randomblob(100); + ROLLBACK; +} + +do_execsql_test 4.1.3 { + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + SELECT count(*) FROM t2; + SELECT count(*) FROM t1; +} {500 10} + +do_test 4.1.4 { + set n [db one { PRAGMA temp.page_count }] + expr ($n >280 && $n < 300) +} 1 + +do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + + CREATE TEMP TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} + +# Test that the temp database is now much bigger than the configured +# cache size (10 pages). +do_test 5.1.2 { + set n [db one { PRAGMA temp.page_count }] + expr ($n > 270 && $n < 290) +} {1} + +do_execsql_test 5.1.3 { + BEGIN; + UPDATE t1 SET a=2; + UPDATE t2 SET a=randomblob(100); + SELECT count(*) FROM t1; + ROLLBACK; +} {1} + +do_execsql_test 5.1.4 { + UPDATE t2 SET a=randomblob(100); + + SELECT * FROM t1; +} {1 2} + +do_execsql_test 5.1.5 { PRAGMA temp.integrity_check } {ok} + +#------------------------------------------------------------------------- +# Test this: +# +# 1. Page is DIRTY at the start of a transaction. +# 2. Page is written out as part of the transaction. +# 3. Page is then read back in. +# 4. Transaction is rolled back. Is the page now clean or dirty? +# +# This actually does work. Step 4 marks the page as clean. But it also +# writes to the database file itself. So marking it clean is correct - +# the page does match the contents of the db file. +# +reset_db + +do_execsql_test 6.1 { + PRAGMA main.cache_size = 10; + PRAGMA temp.cache_size = 10; + + CREATE TEMP TABLE t1(x); + INSERT INTO t1 VALUES('one'); + + CREATE TEMP TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; +} + +do_execsql_test 6.2 { + UPDATE t1 SET x='two'; -- step 1 + BEGIN; + UPDATE t2 SET a=randomblob(100); -- step 2 + SELECT * FROM t1; -- step 3 + ROLLBACK; -- step 4 + + SELECT count(*) FROM t2; + SELECT * FROM t1; +} {two 500 two} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3 db "" +do_execsql_test 7.1 { + PRAGMA auto_vacuum=INCREMENTAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 VALUES(zeroblob(900)); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1 SELECT x FROM t1; + BEGIN; + DELETE FROM t1 WHERE rowid%2; + PRAGMA incremental_vacuum(4); + ROLLBACK; + PRAGMA integrity_check; +} {ok} + +#------------------------------------------------------------------------- +# Try changing the page size using a backup operation when pages are +# stored in main-memory only. +# +reset_db +do_execsql_test 8.1 { + PRAGMA auto_vacuum = OFF; + CREATE TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 ) + INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; +} {13} + +do_test 8.2 { + sqlite3 tmp "" + execsql { + PRAGMA auto_vacuum = OFF; + PRAGMA page_size = 8192; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + PRAGMA page_count; + } tmp +} {10} + +do_test 8.3 { + sqlite3_backup B tmp main db main + B step 5 + B finish +} {SQLITE_READONLY} + +do_test 8.4 { + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + PRAGMA page_size; + } tmp +} {100 ok 8192} + +do_test 8.5 { + tmp eval { UPDATE t1 SET a=randomblob(100) } +} {} + +do_test 8.6 { + sqlite3_backup B tmp main db main + B step 1000 + B finish +} {SQLITE_READONLY} + +tmp close + +#------------------------------------------------------------------------- +# Try inserts and deletes with a large db in auto-vacuum mode. Check +# +foreach {tn mode} { + 1 delete + 2 wal +} { + reset_db + sqlite3 db "" + do_execsql_test 9.$tn.1.1 { + PRAGMA cache_size = 15; + PRAGMA auto_vacuum = 1; + } + execsql "PRAGMA journal_mode = $mode" + + do_execsql_test 9.$tn.1.2 { + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(a); + CREATE INDEX i2 ON tx(b); + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + for {set i 2} {$i<20} {incr i} { + do_execsql_test 9.$tn.$i.1 { DELETE FROM tx WHERE (random()%3)==0 } + + do_execsql_test 9.$tn.$i.2 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.3 { + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + } + + do_execsql_test 9.$tn.$i.4 { PRAGMA integrity_check } ok + + do_execsql_test 9.$tn.$i.5 { + BEGIN; + DELETE FROM tx WHERE (random()%3)==0; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + } + + do_execsql_test 9.$tn.$i.6 { PRAGMA integrity_check } ok + } +} + +#------------------------------------------------------------------------- +# When using mmap mode with a temp file, SQLite must search the cache +# before using a mapped page even when there is no write transaction +# open. For a temp file, the on-disk version may not be up to date. +# +sqlite3 db "" +do_execsql_test 10.0 { + PRAGMA cache_size = 50; + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 2); +} + +do_execsql_test 10.1 { + BEGIN; + WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) + INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; + COMMIT; + INSERT INTO t2 VALUES(3, 4); +} + +if {[permutation]!="journaltest"} { + # The journaltest permutation does not support mmap, so this part of + # the test is omitted. + do_execsql_test 10.2 { PRAGMA mmap_size = 512000 } 512000 +} + +do_execsql_test 10.3 { SELECT * FROM t2 } {1 2 3 4} +do_execsql_test 10.4 { PRAGMA integrity_check } ok + +finish_test + diff --git a/test/tester.tcl b/test/tester.tcl index 2fcae7b5a2..23810d09ee 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1944,6 +1944,12 @@ proc wal_check_journal_mode {testname {db db}} { } } +proc wal_is_capable {} { + ifcapable !wal { return 0 } + if {[permutation]=="journaltest"} { return 0 } + return 1 +} + proc permutation {} { set perm "" catch {set perm $::G(perm:name)} diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test index 3fef187ecd..0d12a6ec43 100644 --- a/test/tkt-2d1a5c67d.test +++ b/test/tkt-2d1a5c67d.test @@ -19,7 +19,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix tkt-2d1a5c67d -ifcapable {!wal || !vtab} {finish_test; return} +ifcapable {!vtab} {finish_test; return} +if {[wal_is_capable]==0} {finish_test; return} for {set ii 1} {$ii<=10} {incr ii} { do_test tkt-2d1a5c67d.1.$ii { diff --git a/test/tkt-313723c356.test b/test/tkt-313723c356.test index 8c08c34976..5325fd2292 100644 --- a/test/tkt-313723c356.test +++ b/test/tkt-313723c356.test @@ -18,7 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -ifcapable !wal { finish_test ; return } +if {![wal_is_capable]} { finish_test ; return } do_execsql_test tkt-313723c356.1 { PRAGMA page_size = 1024; diff --git a/test/tkt-5d863f876e.test b/test/tkt-5d863f876e.test index 86024e300b..9a2fa3f470 100644 --- a/test/tkt-5d863f876e.test +++ b/test/tkt-5d863f876e.test @@ -18,7 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl set ::testprefix tkt-5d863f876e -ifcapable !wal {finish_test ; return } +if {![wal_is_capable]} {finish_test ; return } do_multiclient_test tn { do_test $tn.1 { diff --git a/test/tkt-9d68c883.test b/test/tkt-9d68c883.test index 18dc6ccd67..ba91b39e39 100644 --- a/test/tkt-9d68c883.test +++ b/test/tkt-9d68c883.test @@ -50,4 +50,6 @@ for {set i 0} {$i < 100} {incr i} { } {ok} } +catch { db close } +unregister_devsim finish_test diff --git a/test/zerodamage.test b/test/zerodamage.test index d781ab89a3..a87e50b7b5 100644 --- a/test/zerodamage.test +++ b/test/zerodamage.test @@ -89,7 +89,7 @@ do_test zerodamage-2.1 { concat [file_control_powersafe_overwrite db -1] [set ::max_journal_size] } {0 0 24704} -ifcapable wal { +if {[wal_is_capable]} { # Run a WAL-mode transaction with POWERSAFE_OVERWRITE on to verify that the # WAL file does not get too big. # diff --git a/tool/lemon.c b/tool/lemon.c index 903fe97cfb..2f76465d89 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4402,7 +4402,8 @@ void ReportTable( writeRuleText(out, rp); fprintf(out, " */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ - if( rp2->code==rp->code ){ + if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix + && rp2->codeSuffix==rp->codeSuffix ){ fprintf(out," case %d: /* ", rp2->iRule); writeRuleText(out, rp2); fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++;