diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 2510f33f03..eb547f2149 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -284,8 +284,8 @@ struct sqlite3ota { ota_file *pTargetFd; /* File handle open on target db */ /* The following state variables are used as part of the incremental - ** checkpoint stage (eStage==OTA_STAGE_CKPT). See function otaSetupCkpt() - ** for details. */ + ** checkpoint stage (eStage==OTA_STAGE_CKPT). See comments surrounding + ** function otaSetupCheckpoint() for details. */ u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ u32 mLock; int nFrame; /* Entries in aFrame[] array */ @@ -1730,8 +1730,27 @@ static i64 otaShmChecksum(sqlite3ota *p){ return iRet; } +/* +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3ota.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the OtaState +** object, then the ota handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. +*/ static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){ + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ if( pState==0 ){ p->eStage = 0; if( p->rc==SQLITE_OK ){ @@ -1739,6 +1758,34 @@ static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){ } } + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3ota.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the ota VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3ota.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ if( p->rc==SQLITE_OK ){ int rc2; p->eStage = OTA_STAGE_CAPTURE; @@ -1748,7 +1795,7 @@ static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){ if( p->rc==SQLITE_OK ){ p->eStage = OTA_STAGE_CKPT; - p->nStep = 0; + p->nStep = (pState ? pState->nRow : 0); p->aBuf = otaMalloc(p, p->pgsz); p->iWalCksum = otaShmChecksum(p); } @@ -1759,6 +1806,11 @@ static void otaSetupCheckpoint(sqlite3ota *p, OtaState *pState){ } } +/* +** Called when iAmt bytes are read from offset iOff of the wal file while +** the ota object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. +*/ static int otaCaptureWalRead(sqlite3ota *pOta, i64 iOff, int iAmt){ const u32 mReq = (1<aFrame[pOta->nFrame-1].iDbPage = (u32)(iOff / pOta->pgsz) + 1; return SQLITE_OK; } +/* +** This is called as part of an incremental checkpoint operation. Copy +** a single frame of data from the wal file into the database file, as +** indicated by the OtaFrame object. +*/ static void otaCheckpointFrame(sqlite3ota *p, OtaFrame *pFrame){ sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; sqlite3_file *pDb = p->pTargetFd->pReal; @@ -1921,6 +1983,9 @@ static int otaStepType(sqlite3ota *p, const char **pzMask){ } #ifdef SQLITE_DEBUG +/* +** Assert that column iCol of statement pStmt is named zName. +*/ static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ const char *zCol = sqlite3_column_name(pStmt, iCol); assert( 0==sqlite3_stricmp(zName, zCol) ); @@ -2070,6 +2135,11 @@ static void otaIncrSchemaCookie(sqlite3ota *p){ } } +/* +** Update the contents of the ota_state table within the ota database. The +** value stored in the OTA_STATE_STAGE column is eStage. All other values +** are determined by inspecting the ota handle passed as the first argument. +*/ static void otaSaveState(sqlite3ota *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; @@ -2202,6 +2272,9 @@ int sqlite3ota_step(sqlite3ota *p){ } } +/* +** Free an OtaState object allocated by otaLoadState(). +*/ static void otaFreeState(OtaState *p){ if( p ){ sqlite3_free(p->zTbl); @@ -2278,13 +2351,28 @@ static OtaState *otaLoadState(sqlite3ota *p){ return pRet; } +/* +** Compare strings z1 and z2, returning 0 if they are identical, or non-zero +** otherwise. Either or both argument may be NULL. Two NULL values are +** considered equal, and NULL is considered distinct from all other values. +*/ static int otaStrCompare(const char *z1, const char *z2){ if( z1==0 && z2==0 ) return 0; if( z1==0 || z2==0 ) return 1; return (sqlite3_stricmp(z1, z2)!=0); } -static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ +/* +** This function is called as part of sqlite3ota_open() when initializing +** an ota handle in OAL stage. If the ota update has not started (i.e. +** the ota_state table was empty) it is a no-op. Otherwise, it arranges +** things so that the next call to sqlite3ota_step() continues on from +** where the previous ota handle left off. +** +** If an error occurs, an error code and error message are left in the +** ota handle passed as the first argument. +*/ +static void otaSetupOal(sqlite3ota *p, OtaState *pState){ assert( p->rc==SQLITE_OK ); if( pState->zTbl ){ OtaObjIter *pIter = &p->objiter; @@ -2323,6 +2411,12 @@ static void otaDeleteOalFile(sqlite3ota *p){ sqlite3_free(zOal); } +/* +** Allocate a private ota VFS for the ota handle passed as the only +** argument. This VFS will be used unless the call to sqlite3ota_open() +** specified a URI with a vfs=? option in place of a target database +** file name. +*/ static void otaCreateVfs(sqlite3ota *p){ int rnd; char zRnd[64]; @@ -2338,6 +2432,10 @@ static void otaCreateVfs(sqlite3ota *p){ } } +/* +** Destroy the private VFS created for the ota handle passed as the only +** argument by an earlier call to otaCreateVfs(). +*/ static void otaDeleteVfs(sqlite3ota *p){ if( p->zVfsName ){ sqlite3ota_destroy_vfs(p->zVfsName); @@ -2423,13 +2521,12 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ } if( p->rc==SQLITE_OK ){ - otaLoadTransactionState(p, pState); + otaSetupOal(p, pState); } }else if( p->eStage==OTA_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==OTA_STAGE_CKPT ){ otaSetupCheckpoint(p, pState); - p->nStep = pState->nRow; }else if( p->eStage==OTA_STAGE_DONE ){ p->rc = SQLITE_DONE; }else{ @@ -2794,7 +2891,7 @@ static int otaVfsDeviceCharacteristics(sqlite3_file *pFile){ } /* -** Shared-memory methods are all pass-thrus. +** Take or release a shared-memory lock. */ static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ ota_file *p = (ota_file*)pFile; @@ -2832,6 +2929,9 @@ static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ return rc; } +/* +** Obtain a pointer to a mapping of a single 32KiB page of the *-shm file. +*/ static int otaVfsShmMap( sqlite3_file *pFile, int iRegion, @@ -2891,6 +2991,9 @@ static void otaVfsShmBarrier(sqlite3_file *pFile){ p->pReal->pMethods->xShmBarrier(p->pReal); } +/* +** The xShmUnmap method. +*/ static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ ota_file *p = (ota_file*)pFile; int rc = SQLITE_OK; @@ -2905,6 +3008,12 @@ static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ return rc; } +/* +** Given that zWal points to a buffer containing a wal file name passed to +** either the xOpen() or xAccess() VFS method, return a pointer to the +** file-handle opened by the same database connection on the corresponding +** database file. +*/ static ota_file *otaFindMaindb(ota_vfs *pOtaVfs, const char *zWal){ ota_file *pDb; sqlite3_mutex_enter(pOtaVfs->mutex); @@ -3146,10 +3255,17 @@ static int otaVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return pRealVfs->xCurrentTime(pRealVfs, pTimeOut); } +/* +** No-op. +*/ static int otaVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ return 0; } +/* +** Deregister and destroy an OTA vfs created by an earlier call to +** sqlite3ota_create_vfs(). +*/ void sqlite3ota_destroy_vfs(const char *zName){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); if( pVfs && pVfs->xOpen==otaVfsOpen ){ @@ -3159,6 +3275,11 @@ void sqlite3ota_destroy_vfs(const char *zName){ } } +/* +** Create an OTA VFS named zName that accesses the underlying file-system +** via existing VFS zParent. The new object is registered as a non-default +** VFS with SQLite before returning. +*/ int sqlite3ota_create_vfs(const char *zName, const char *zParent){ /* Template for VFS */ diff --git a/ext/ota/sqlite3ota.h b/ext/ota/sqlite3ota.h index 7d3d1e9494..814cd7f0ed 100644 --- a/ext/ota/sqlite3ota.h +++ b/ext/ota/sqlite3ota.h @@ -310,6 +310,10 @@ int sqlite3ota_close(sqlite3ota *pOta, char **pzErrmsg); sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta); /* +** Create an OTA VFS named zName that accesses the underlying file-system +** via existing VFS zParent. The new object is registered as a non-default +** VFS with SQLite before returning. +** ** Part of the OTA implementation uses a custom VFS object. Usually, this ** object is created and deleted automatically by OTA. ** diff --git a/manifest b/manifest index a022fbaa87..2d0cfbd4d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\smutex\sused\sto\sprotect\sthe\slinked\slist\sof\sall\smain\sdatabase\sfiles\sopened\sby\sa\ssingle\sota\svfs\sis\sallocated. -D 2015-02-19T19:59:35.500 +C Add\ssome\smissing\scomments\sand\sfix\sother\sminor\scode\sissues\sin\ssqlite3ota.c. +D 2015-02-20T14:36:16.318 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,8 +137,8 @@ F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b F ext/ota/otaA.test ef4bfa8cfd4ed814ae86f7457b64aa2f18c90171 F ext/ota/otafault.test 8c43586c2b96ca16bbce00b5d7e7d67316126db8 F ext/ota/otafault2.test fa202a98ca221faec318f3e5c5f39485b1256561 -F ext/ota/sqlite3ota.c 692de0a8c43dbe7a98e0cb6760c55635c3d58b5a -F ext/ota/sqlite3ota.h 69106b04616f4e7e565aa4dc2092a2f095212cc2 +F ext/ota/sqlite3ota.c 0cd5505ebd0e506ac17a2bc8b63623ab93aa0b4e +F ext/ota/sqlite3ota.h 052d87068751810a9dfba1a48954271501bb728f F ext/ota/test_ota.c 9ec6ea945282f65f67f0e0468dad79a489818f44 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f @@ -1257,7 +1257,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 60e0a46b82dd9c704e8aa977d1ccdd73d388422f -R 24048fcc0c09eb0db3eb9c1d038269b2 +P 9c8682d6650a94e11f9bec5baff69ed9668874fa +R 0b2e9b82020c9eb4b7288b4840940ccc U dan -Z 33143372b71a8b702a8cbb544384d97d +Z c1cac3a09c083c8da890457fb5b17289 diff --git a/manifest.uuid b/manifest.uuid index de388b6f0e..c37022e05d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c8682d6650a94e11f9bec5baff69ed9668874fa \ No newline at end of file +718fd8b673d6557dd0eaad03e6a3332b5490afbf \ No newline at end of file