From a58f26f93f77ae83021d52ff76738f27a59596b9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 16 Nov 2010 18:56:51 +0000 Subject: [PATCH 1/6] Add experimental command "PRAGMA wal_blocking_checkpoint", which uses the busy-handler to block until all readers have finished in order to ensure the next writer will be able to wrap around to the start of the log file. FossilOrigin-Name: 7e3fc2c833a5baa08820c499867b6902bdc2ed5a --- manifest | 48 ++++++++---------- manifest.uuid | 2 +- src/btree.c | 11 ++++- src/btree.h | 2 +- src/main.c | 9 ++-- src/pager.c | 10 +++- src/pager.h | 2 +- src/pragma.c | 10 +++- src/sqliteInt.h | 2 +- src/vdbe.c | 6 +-- src/wal.c | 97 +++++++++++++++++++++++++++--------- src/wal.h | 4 +- test/wal5.test | 118 ++++++++++++++++++++++++++++++++++++++++++++ test/wal_common.tcl | 2 + 14 files changed, 256 insertions(+), 67 deletions(-) create mode 100644 test/wal5.test diff --git a/manifest b/manifest index d06d475d13..a133dac267 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Use\sthe\sestimated\snumber\sof\srows\scomputed\sfor\ssubqueries\sin\sthe\scost\ncomputations\sfor\souter\squeries. -D 2010-11-16T02:49:16 +C Add\sexperimental\scommand\s"PRAGMA\swal_blocking_checkpoint",\swhich\suses\sthe\sbusy-handler\sto\sblock\suntil\sall\sreaders\shave\sfinished\sin\sorder\sto\sensure\sthe\snext\swriter\swill\sbe\sable\sto\swrap\saround\sto\sthe\sstart\sof\sthe\slog\sfile. +D 2010-11-16T18:56:51 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,8 +119,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c d5b0137bc20327af08c14772227cc35134839c30 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 3edab36d03d86c200cb9551467410f975d510aa9 -F src/btree.h 2d1a83ad509047e8cc314fda7e054f99ff52414d +F src/btree.c 444aae4fc60cc57d6c97615358e1020f6884cca6 +F src/btree.h d1144d38d790a8b7b2e215043f8d068f4f37de07 F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0 F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d F src/callback.c a1d1b1c9c85415dff013af033e2fed9c8382d33b @@ -144,7 +141,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e -F src/main.c 335d2c7ecb093db7b662d9325fe68fb9087a4814 +F src/main.c 89c658ae9a610a61ff856a110bda50606e9227d6 F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -165,13 +162,13 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c de5be4cdbf3d07018059934eaf7e5d8d594a895c F src/os_win.c 2f90f7bdec714fad51cd31b4ecad3cc1b4bb5aad -F src/pager.c 067ae23d7a370eea6bd529848331c63879570adc -F src/pager.h 8167a1e720d0b7a2790079007128e594010220ad +F src/pager.c 7f7587c2f11126d13ee1925ac8960a9e7ab13e8a +F src/pager.h ad7d8db0fbcee7546dbc02ffe0d0d44ea868ef52 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 09d38c44ab275db581f7a2f6ff8b9bc7f8c0faaa F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c e9578a3beac26f229ee558a4e16c863f2498185f -F src/pragma.c 216d12e4546e65ca6cfcd3221e64573889ae8f34 +F src/pragma.c 66a8b53d1e74635011fbb0bb54b7ecc402684bae F src/prepare.c c2b318037d626fed27905c9446730b560637217a F src/printf.c 8ae5082dd38a1b5456030c3755ec3a392cd51506 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -181,7 +178,7 @@ F src/select.c 550d67688f5e8bc8022faf6d014838afba1415af F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056 F src/sqlite.h.in f47e09412fc9a129f759fa4d96ef21f4b3d529eb F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h dd28f6138c74cf4833e032a989b6ff7885798cf6 +F src/sqliteInt.h fe1cb073b2707001985f06dee9ee256247e4d0ce F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -230,7 +227,7 @@ F src/update.c 227e6cd512108b84f69421fc6c7aa1b83d60d6e0 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c cd78524566fe45671863eee78685969a4bfd4e4c F src/vacuum.c 924bd1bcee2dfb05376f79845bd3b4cec7b54b2f -F src/vdbe.c e1aa917961e69f71c80f46ce231b496d3c841ae1 +F src/vdbe.c b86b09beb3dcf2e6d5922acee48b8a1c16b68bfd F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2 F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4 F src/vdbeapi.c 5368714fa750270cf6430160287c21adff44582d @@ -239,8 +236,8 @@ F src/vdbeblob.c e0ce3c54cc0c183af2ec67b63a289acf92251df4 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30 -F src/wal.c f26b8d297bd11cb792e609917f9d4c6718ac8e0e -F src/wal.h c1aac6593a0b02b15dc625987e619edeab39292e +F src/wal.c 400624ce58acce44f0bf0d47ed2f435da290fb04 +F src/wal.h d5bbc11242d7fd14e9dc6a74f68d3ccaf01a9e48 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c d5cc65f51661a038a2c6a663a945d5cf4c277b81 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -830,7 +827,8 @@ F test/wal.test 70227190e713b3e7eb2a7d5ec3510b66db01f327 F test/wal2.test c794b8b257af54190bb913678ad3984cbf3311b9 F test/wal3.test 957a5f2a8fe8a6ff01de1a15285ecf2f376fcaf8 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal_common.tcl 895d76138043b86bdccf36494054bdabcf65837b +F test/wal5.test e0f1abdff4f76d3a8531f5d0f4cb237e5eff891c +F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 F test/walcksum.test a37b36375c595e61bdb7e1ec49b5f0979b6fc7ce @@ -889,14 +887,10 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d52b593978aa1776af7aeb957c4f8df0c5cb7e43 -R d49dbce342d6074d2c08ca15a1061dc2 -U drh -Z 1d0a2a683be8d99ef69f89af3e0fc4c3 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFM4fEvoxKgR168RlERAtLBAJ0duCZxNY8VlhlVLo/NezKrqI5noACdE3GU -LDKZgeEbRVz6H1/mucVVNUQ= -=B2su ------END PGP SIGNATURE----- +P 56bbc539246a6dc9f1ae1edb898db7a4f6f6d322 +R 75a04ae738d792c76538f98d6f9ad655 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z b1cbb492cf90106bd968e524fdd11e16 diff --git a/manifest.uuid b/manifest.uuid index 5f256ca04e..936ce08f7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56bbc539246a6dc9f1ae1edb898db7a4f6f6d322 \ No newline at end of file +7e3fc2c833a5baa08820c499867b6902bdc2ed5a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7e8c39feb6..94100f48fc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7935,8 +7935,15 @@ int sqlite3BtreeIsInTrans(Btree *p){ ** ** Return SQLITE_LOCKED if this or any other connection has an open ** transaction on the shared-cache the argument Btree is connected to. +** +** If parameter bBlock is true, then the layers below invoke the +** busy-handler callback while waiting for readers to release locks so +** that the entire WAL can be checkpointed. If it is false, then as +** much as possible of the WAL is checkpointed without waiting for readers +** to finish. bBlock is true for "PRAGMA wal_blocking_checkpoint" and false +** for "PRAGMA wal_checkpoint". */ -int sqlite3BtreeCheckpoint(Btree *p){ +int sqlite3BtreeCheckpoint(Btree *p, int bBlock){ int rc = SQLITE_OK; if( p ){ BtShared *pBt = p->pBt; @@ -7944,7 +7951,7 @@ int sqlite3BtreeCheckpoint(Btree *p){ if( pBt->inTransaction!=TRANS_NONE ){ rc = SQLITE_LOCKED; }else{ - rc = sqlite3PagerCheckpoint(pBt->pPager); + rc = sqlite3PagerCheckpoint(pBt->pPager, bBlock); } sqlite3BtreeLeave(p); } diff --git a/src/btree.h b/src/btree.h index 39af03f961..ce86cdabb1 100644 --- a/src/btree.h +++ b/src/btree.h @@ -207,7 +207,7 @@ void sqlite3BtreeCursorList(Btree*); #endif #ifndef SQLITE_OMIT_WAL - int sqlite3BtreeCheckpoint(Btree*); + int sqlite3BtreeCheckpoint(Btree*, int); #endif /* diff --git a/src/main.c b/src/main.c index 25216070d6..7ce43aec5c 100644 --- a/src/main.c +++ b/src/main.c @@ -1361,7 +1361,7 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ - rc = sqlite3Checkpoint(db, iDb); + rc = sqlite3Checkpoint(db, iDb, 0); sqlite3Error(db, rc, 0); } rc = sqlite3ApiExit(db, rc); @@ -1387,8 +1387,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ ** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are ** checkpointed. If an error is encountered it is returned immediately - ** no attempt is made to checkpoint any remaining databases. +** +** Parameter bBlock is true for a blocking-checkpoint, false for an +** ordinary, non-blocking, checkpoint. */ -int sqlite3Checkpoint(sqlite3 *db, int iDb){ +int sqlite3Checkpoint(sqlite3 *db, int iDb, int bBlock){ int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterate through attached dbs */ @@ -1396,7 +1399,7 @@ int sqlite3Checkpoint(sqlite3 *db, int iDb){ for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt); + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, bBlock); } } diff --git a/src/pager.c b/src/pager.c index b774af3a5f..29f3bf1eb1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6516,13 +6516,19 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ #ifndef SQLITE_OMIT_WAL /* -** This function is called when the user invokes "PRAGMA checkpoint". +** This function is called when the user invokes "PRAGMA wal_checkpoint", +** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() +** or wal_blocking_checkpoint() API functions. +** +** Parameter bBlock is true for a blocking-checkpoint, false for an +** ordinary, non-blocking, checkpoint. */ -int sqlite3PagerCheckpoint(Pager *pPager){ +int sqlite3PagerCheckpoint(Pager *pPager, int bBlock){ int rc = SQLITE_OK; if( pPager->pWal ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; rc = sqlite3WalCheckpoint(pPager->pWal, + (bBlock ? pPager->xBusyHandler : 0), pPager->pBusyHandlerArg, (pPager->noSync ? 0 : pPager->sync_flags), pPager->pageSize, zBuf ); diff --git a/src/pager.h b/src/pager.h index c12afa7b8b..e0396e99aa 100644 --- a/src/pager.h +++ b/src/pager.h @@ -138,7 +138,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); -int sqlite3PagerCheckpoint(Pager *pPager); +int sqlite3PagerCheckpoint(Pager *pPager, int); int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); diff --git a/src/pragma.c b/src/pragma.c index 362e77f29e..7324e1bde8 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1394,12 +1394,18 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* ** PRAGMA [database.]wal_checkpoint + ** PRAGMA [database.]wal_blocking_checkpoint ** ** Checkpoint the database. */ - if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ + if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 + || sqlite3StrICmp(zLeft, "wal_blocking_checkpoint")==0 + ){ + int bBlock = (zLeft[14]!=0); + int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); + assert( bBlock==(sqlite3StrICmp(zLeft, "wal_checkpoint")!=0) ); if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeAddOp3(v, OP_Checkpoint, pId2->z?iDb:SQLITE_MAX_ATTACHED, 0, 0); + sqlite3VdbeAddOp2(v, OP_Checkpoint, iBt, bBlock); }else /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c02a0e4487..8b4fe599eb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3036,7 +3036,7 @@ CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); const char *sqlite3JournalModename(int); -int sqlite3Checkpoint(sqlite3*, int); +int sqlite3Checkpoint(sqlite3*, int, int); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); /* Declarations for functions in fkey.c. All of these are replaced by diff --git a/src/vdbe.c b/src/vdbe.c index 02d1a406c9..dcff83f17a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5216,13 +5216,13 @@ case OP_AggFinal: { } #ifndef SQLITE_OMIT_WAL -/* Opcode: Checkpoint P1 * * * * +/* Opcode: Checkpoint P1 P2 * * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. +** WAL mode. If P2 is non-zero, this is a blocking checkpoint. */ case OP_Checkpoint: { - rc = sqlite3Checkpoint(db, pOp->p1); + rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2); break; }; #endif diff --git a/src/wal.c b/src/wal.c index 3b217908b7..5ff9f227e0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1523,6 +1523,26 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ return rc; } +/* +** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and +** n. If the attempt fails and parameter xBusy is not NULL, then it is a +** busy-handler function. Invoke it and retry the lock until either the +** lock is successfully obtained or the busy-handler returns 0. +*/ +static int walBusyLock( + Wal *pWal, /* WAL connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int lockIdx, /* Offset of first byte to lock */ + int n /* Number of bytes to lock */ +){ + int rc; + do { + rc = walLockExclusive(pWal, lockIdx, n); + }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); + return rc; +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1556,6 +1576,8 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ */ static int walCheckpoint( Wal *pWal, /* Wal connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ int nBuf, /* Size of zBuf in bytes */ u8 *zBuf /* Temporary buffer to use */ @@ -1593,27 +1615,29 @@ static int walCheckpoint( ** overwrite database pages that are in use by active readers and thus ** cannot be backfilled from the WAL. */ - mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - pInfo = walCkptInfo(pWal); - for(i=1; iaReadMark[i]; - if( mxSafeFrame>=y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = READMARK_NOT_USED; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - }else{ - goto walcheckpoint_out; + do { + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + pInfo = walCkptInfo(pWal); + for(i=1; iaReadMark[i]; + if( mxSafeFrame>=y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = READMARK_NOT_USED; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + }else{ + goto walcheckpoint_out; + } } } - } + }while( xBusy && mxSafeFramehdr.mxFrame && xBusy(pBusyArg) ); if( pInfo->nBackfillnBackfill; @@ -1666,10 +1690,19 @@ static int walCheckpoint( /* Release the reader lock held while backfilling */ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); - }else if( rc==SQLITE_BUSY ){ + + if( xBusy && rc==SQLITE_OK && pWal->hdr.mxFrame==mxSafeFrame ){ + assert( pWal->writeLock ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } + } + } + + if( rc==SQLITE_BUSY ){ /* Reset the return code so as not to report a checkpoint failure - ** just because active readers prevent any backfill. - */ + ** just because there are active readers. */ rc = SQLITE_OK; } @@ -1704,7 +1737,7 @@ int sqlite3WalClose( if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf); + rc = sqlite3WalCheckpoint(pWal, 0, 0, sync_flags, nBuf, zBuf); if( rc==SQLITE_OK ){ isDelete = 1; } @@ -2619,9 +2652,14 @@ int sqlite3WalFrames( ** ** Obtain a CHECKPOINT lock and then backfill as much information as ** we can from WAL into the database. +** +** If parameter xBusy is not NULL, it is a pointer to a busy-handler +** callback. In this case this function runs a blocking checkpoint. */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ int nBuf, /* Size of temporary buffer */ u8 *zBuf /* Temporary buffer to use */ @@ -2630,6 +2668,7 @@ int sqlite3WalCheckpoint( int isChanged = 0; /* True if a new wal-index header is loaded */ assert( pWal->ckptLock==0 ); + assert( pWal->writeLock==0 ); WALTRACE(("WAL%p: checkpoint begins\n", pWal)); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); @@ -2641,10 +2680,21 @@ int sqlite3WalCheckpoint( } pWal->ckptLock = 1; + /* If this is a blocking-checkpoint, then obtain the write-lock as well + ** to prevent any writers from running while the checkpoint is underway. + ** This has to be done before the call to walIndexReadHdr() below. + */ + if( xBusy ){ + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ) pWal->writeLock = 1; + } + /* Copy data from the log to the database file. */ - rc = walIndexReadHdr(pWal, &isChanged); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf); + rc = walIndexReadHdr(pWal, &isChanged); + } + if( rc==SQLITE_OK ){ + rc = walCheckpoint(pWal, xBusy, pBusyArg, sync_flags, nBuf, zBuf); } if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was @@ -2657,6 +2707,7 @@ int sqlite3WalCheckpoint( } /* Release the locks. */ + sqlite3WalEndWriteTransaction(pWal); walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); diff --git a/src/wal.h b/src/wal.h index 35f695c88a..e9c614fe85 100644 --- a/src/wal.h +++ b/src/wal.h @@ -32,7 +32,7 @@ # define sqlite3WalSavepoint(y,z) # define sqlite3WalSavepointUndo(y,z) 0 # define sqlite3WalFrames(u,v,w,x,y,z) 0 -# define sqlite3WalCheckpoint(u,v,w,x) 0 +# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 @@ -86,6 +86,8 @@ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ int sqlite3WalCheckpoint( Wal *pWal, /* Write-ahead log connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ int nBuf, /* Size of buffer nBuf */ u8 *zBuf /* Temporary buffer to use */ diff --git a/test/wal5.test b/test/wal5.test new file mode 100644 index 0000000000..24d6353a62 --- /dev/null +++ b/test/wal5.test @@ -0,0 +1,118 @@ +# 2010 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of "blocking-checkpoint" +# operations. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +ifcapable !wal {finish_test ; return } + +set testprefix wal5 + +do_multiclient_test tn { + + proc db_page_count {} { expr [file size test.db] / 1024 } + proc wal_page_count {} { wal_frame_count test.db-wal 1024 } + + set ::nBusyHandler 0 + set ::busy_handler_script "" + proc busyhandler {n} { + incr ::nBusyHandler + eval $::busy_handler_script + return 0 + } + + proc reopen_all {} { + code1 {db close} + code2 {db2 close} + code3 {db3 close} + code1 {sqlite3 db test.db} + code2 {sqlite3 db2 test.db} + code3 {sqlite3 db3 test.db} + sql1 { PRAGMA synchronous = NORMAL } + code1 { db busy busyhandler } + } + + do_test 1.$tn.1 { + reopen_all + sql1 { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, zeroblob(1200)); + INSERT INTO t1 VALUES(2, zeroblob(1200)); + INSERT INTO t1 VALUES(3, zeroblob(1200)); + } + expr [file size test.db] / 1024 + } {2} + + # Have connection 2 grab a read-lock on the current snapshot. + do_test 1.$tn.2 { sql2 { BEGIN; SELECT x FROM t1 } } {1 2 3} + + # Attempt a checkpoint. + do_test 1.$tn.3 { + sql1 { PRAGMA wal_checkpoint } + list [db_page_count] [wal_page_count] + } {5 9} + + # Write to the db again. The log cannot wrap because of the lock still + # held by connection 2. The busy-handler has not yet been invoked. + do_test 1.$tn.4 { + sql1 { INSERT INTO t1 VALUES(4, zeroblob(1200)) } + list [db_page_count] [wal_page_count] $::nBusyHandler + } {5 12 0} + + # Now do a blocking-checkpoint. Set the busy-handler up so that connection + # 2 releases its lock on the 6th invocation. The checkpointer should then + # proceed to checkpoint the entire log file. Next write should go to the + # start of the log file. + # + set ::busy_handler_script { if {$n==5} { sql2 COMMIT } } + do_test 1.$tn.5 { + sql1 { PRAGMA wal_blocking_checkpoint } + list [db_page_count] [wal_page_count] $::nBusyHandler + } {6 12 6} + do_test 1.$tn.6 { + set ::nBusyHandler 0 + sql1 { INSERT INTO t1 VALUES(5, zeroblob(1200)) } + list [db_page_count] [wal_page_count] $::nBusyHandler + } {6 12 0} + + do_test 1.$tn.7 { + reopen_all + list [db_page_count] [wal_page_count] $::nBusyHandler + } {7 0 0} + + do_test 1.$tn.8 { sql2 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5} + do_test 1.$tn.9 { + sql1 { INSERT INTO t1 VALUES(6, zeroblob(1200)) } + list [db_page_count] [wal_page_count] $::nBusyHandler + } {7 5 0} + do_test 1.$tn.10 { sql3 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5 6} + + set ::busy_handler_script { + if {$n==5} { sql2 COMMIT } + if {$n==6} { set ::db_file_size [db_page_count] } + if {$n==7} { sql3 COMMIT } + } + do_test 1.$tn.11 { + sql1 { PRAGMA wal_blocking_checkpoint } + list [db_page_count] [wal_page_count] $::nBusyHandler + } {10 5 8} + do_test 1.$tn.12 { set ::db_file_size } 10 +} + +finish_test diff --git a/test/wal_common.tcl b/test/wal_common.tcl index a5b165d5ec..917ad598f6 100644 --- a/test/wal_common.tcl +++ b/test/wal_common.tcl @@ -18,7 +18,9 @@ proc wal_file_size {nFrame pgsz} { } proc wal_frame_count {zFile pgsz} { + if {[file exists $zFile]==0} { return 0 } set f [file size $zFile] + if {$f < 32} { return 0 } expr {($f - 32) / ($pgsz+24)} } From cdc1f049bf19766b4847cb59ec87b680e2982671 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Nov 2010 12:11:05 +0000 Subject: [PATCH 2/6] Modify the interface to the blocking wal-checkpoint functionality. FossilOrigin-Name: 72787c010c8944e8fcf9c98aa4482f129142d8e9 --- manifest | 37 ++++++++++------------ manifest.uuid | 2 +- src/btree.c | 11 ++----- src/btree.h | 2 +- src/main.c | 39 +++++++++++++++++------ src/pager.c | 13 ++++---- src/pager.h | 2 +- src/pragma.c | 26 +++++++++++----- src/sqlite.h.in | 83 +++++++++++++++++++++++++++++++++++++++++++++++++ src/sqliteInt.h | 2 +- src/vdbe.c | 26 ++++++++++++++-- src/wal.c | 48 ++++++++++++++++++---------- src/wal.h | 33 +++++++++++--------- test/wal5.test | 5 +-- 14 files changed, 236 insertions(+), 93 deletions(-) diff --git a/manifest b/manifest index a133dac267..7dd2eb66e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\scommand\s"PRAGMA\swal_blocking_checkpoint",\swhich\suses\sthe\sbusy-handler\sto\sblock\suntil\sall\sreaders\shave\sfinished\sin\sorder\sto\sensure\sthe\snext\swriter\swill\sbe\sable\sto\swrap\saround\sto\sthe\sstart\sof\sthe\slog\sfile. -D 2010-11-16T18:56:51 +C Modify\sthe\sinterface\sto\sthe\sblocking\swal-checkpoint\sfunctionality. +D 2010-11-18T12:11:05 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -119,8 +119,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c d5b0137bc20327af08c14772227cc35134839c30 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 444aae4fc60cc57d6c97615358e1020f6884cca6 -F src/btree.h d1144d38d790a8b7b2e215043f8d068f4f37de07 +F src/btree.c d90149f6e0a6f715b58b272ef1028fa249a2a088 +F src/btree.h 1d62748eb7d129292782cf65b891b85cbfa024d4 F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0 F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d F src/callback.c a1d1b1c9c85415dff013af033e2fed9c8382d33b @@ -141,7 +141,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e -F src/main.c 89c658ae9a610a61ff856a110bda50606e9227d6 +F src/main.c 91465f2658911ddb51be89e7b8ee01af8584308f F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -162,13 +162,13 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e F src/os_unix.c de5be4cdbf3d07018059934eaf7e5d8d594a895c F src/os_win.c 2f90f7bdec714fad51cd31b4ecad3cc1b4bb5aad -F src/pager.c 7f7587c2f11126d13ee1925ac8960a9e7ab13e8a -F src/pager.h ad7d8db0fbcee7546dbc02ffe0d0d44ea868ef52 +F src/pager.c b46a78a196d99bc855eec3c602777a1bc8db5122 +F src/pager.h e2485f2f2fa5264f2bb68d1783c149d3d57d3637 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 09d38c44ab275db581f7a2f6ff8b9bc7f8c0faaa F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c e9578a3beac26f229ee558a4e16c863f2498185f -F src/pragma.c 66a8b53d1e74635011fbb0bb54b7ecc402684bae +F src/pragma.c f843c877845ddbb911f10eea50c9290bc8354b03 F src/prepare.c c2b318037d626fed27905c9446730b560637217a F src/printf.c 8ae5082dd38a1b5456030c3755ec3a392cd51506 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,9 +176,9 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 550d67688f5e8bc8022faf6d014838afba1415af F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056 -F src/sqlite.h.in f47e09412fc9a129f759fa4d96ef21f4b3d529eb +F src/sqlite.h.in 4645a3bddf4481fcc9422ba41acf4e71c1c81e22 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h fe1cb073b2707001985f06dee9ee256247e4d0ce +F src/sqliteInt.h 4e7045f17606296bc8e7898d69567fc3cd06b761 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -227,7 +227,7 @@ F src/update.c 227e6cd512108b84f69421fc6c7aa1b83d60d6e0 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c cd78524566fe45671863eee78685969a4bfd4e4c F src/vacuum.c 924bd1bcee2dfb05376f79845bd3b4cec7b54b2f -F src/vdbe.c b86b09beb3dcf2e6d5922acee48b8a1c16b68bfd +F src/vdbe.c 4bec828e70654c698ef843c29b557bee2c8a0a00 F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2 F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4 F src/vdbeapi.c 5368714fa750270cf6430160287c21adff44582d @@ -236,8 +236,8 @@ F src/vdbeblob.c e0ce3c54cc0c183af2ec67b63a289acf92251df4 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30 -F src/wal.c 400624ce58acce44f0bf0d47ed2f435da290fb04 -F src/wal.h d5bbc11242d7fd14e9dc6a74f68d3ccaf01a9e48 +F src/wal.c 8eca619a28a70a667c913e5927131250836377a2 +F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c d5cc65f51661a038a2c6a663a945d5cf4c277b81 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -827,7 +827,7 @@ F test/wal.test 70227190e713b3e7eb2a7d5ec3510b66db01f327 F test/wal2.test c794b8b257af54190bb913678ad3984cbf3311b9 F test/wal3.test 957a5f2a8fe8a6ff01de1a15285ecf2f376fcaf8 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test e0f1abdff4f76d3a8531f5d0f4cb237e5eff891c +F test/wal5.test 4e2854d7584dd97a73e7ce0f47bcbbe5c592fe29 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -887,10 +887,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 56bbc539246a6dc9f1ae1edb898db7a4f6f6d322 -R 75a04ae738d792c76538f98d6f9ad655 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P 7e3fc2c833a5baa08820c499867b6902bdc2ed5a +R 1672f1722e4277222ee64bddb8543abf U dan -Z b1cbb492cf90106bd968e524fdd11e16 +Z 2e080f5eeb72d6f9662d3b3c376a7b7c diff --git a/manifest.uuid b/manifest.uuid index 936ce08f7a..3807e38807 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e3fc2c833a5baa08820c499867b6902bdc2ed5a \ No newline at end of file +72787c010c8944e8fcf9c98aa4482f129142d8e9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 94100f48fc..f354b43558 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7936,14 +7936,9 @@ int sqlite3BtreeIsInTrans(Btree *p){ ** Return SQLITE_LOCKED if this or any other connection has an open ** transaction on the shared-cache the argument Btree is connected to. ** -** If parameter bBlock is true, then the layers below invoke the -** busy-handler callback while waiting for readers to release locks so -** that the entire WAL can be checkpointed. If it is false, then as -** much as possible of the WAL is checkpointed without waiting for readers -** to finish. bBlock is true for "PRAGMA wal_blocking_checkpoint" and false -** for "PRAGMA wal_checkpoint". +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -int sqlite3BtreeCheckpoint(Btree *p, int bBlock){ +int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( p ){ BtShared *pBt = p->pBt; @@ -7951,7 +7946,7 @@ int sqlite3BtreeCheckpoint(Btree *p, int bBlock){ if( pBt->inTransaction!=TRANS_NONE ){ rc = SQLITE_LOCKED; }else{ - rc = sqlite3PagerCheckpoint(pBt->pPager, bBlock); + rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt); } sqlite3BtreeLeave(p); } diff --git a/src/btree.h b/src/btree.h index ce86cdabb1..4bfc130ef1 100644 --- a/src/btree.h +++ b/src/btree.h @@ -207,7 +207,7 @@ void sqlite3BtreeCursorList(Btree*); #endif #ifndef SQLITE_OMIT_WAL - int sqlite3BtreeCheckpoint(Btree*, int); + int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); #endif /* diff --git a/src/main.c b/src/main.c index 7ce43aec5c..4333a5873b 100644 --- a/src/main.c +++ b/src/main.c @@ -1340,19 +1340,29 @@ void *sqlite3_wal_hook( #endif } - /* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. +** Checkpoint database zDb. */ -int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ +int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ #ifdef SQLITE_OMIT_WAL return SQLITE_OK; #else int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE + && eMode!=SQLITE_CHECKPOINT_FULL + && eMode!=SQLITE_CHECKPOINT_RESTART + ){ + return SQLITE_MISUSE; + } + sqlite3_mutex_enter(db->mutex); if( zDb && zDb[0] ){ iDb = sqlite3FindDbName(db, zDb); @@ -1361,7 +1371,7 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ - rc = sqlite3Checkpoint(db, iDb, 0); + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); sqlite3Error(db, rc, 0); } rc = sqlite3ApiExit(db, rc); @@ -1370,6 +1380,16 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ #endif } + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); +} + #ifndef SQLITE_OMIT_WAL /* ** Run a checkpoint on database iDb. This is a no-op if database iDb is @@ -1388,10 +1408,9 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ ** checkpointed. If an error is encountered it is returned immediately - ** no attempt is made to checkpoint any remaining databases. ** -** Parameter bBlock is true for a blocking-checkpoint, false for an -** ordinary, non-blocking, checkpoint. +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -int sqlite3Checkpoint(sqlite3 *db, int iDb, int bBlock){ +int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterate through attached dbs */ @@ -1399,7 +1418,7 @@ int sqlite3Checkpoint(sqlite3 *db, int iDb, int bBlock){ for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, bBlock); + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); } } diff --git a/src/pager.c b/src/pager.c index 29f3bf1eb1..bb9f6ddf02 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6520,17 +6520,16 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() ** or wal_blocking_checkpoint() API functions. ** -** Parameter bBlock is true for a blocking-checkpoint, false for an -** ordinary, non-blocking, checkpoint. +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -int sqlite3PagerCheckpoint(Pager *pPager, int bBlock){ +int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( pPager->pWal ){ - u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3WalCheckpoint(pPager->pWal, - (bBlock ? pPager->xBusyHandler : 0), pPager->pBusyHandlerArg, + rc = sqlite3WalCheckpoint(pPager->pWal, eMode, + pPager->xBusyHandler, pPager->pBusyHandlerArg, (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, zBuf + pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt ); } return rc; diff --git a/src/pager.h b/src/pager.h index e0396e99aa..27524f0217 100644 --- a/src/pager.h +++ b/src/pager.h @@ -138,7 +138,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); -int sqlite3PagerCheckpoint(Pager *pPager, int); +int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); diff --git a/src/pragma.c b/src/pragma.c index 7324e1bde8..fb7fe2c920 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1393,19 +1393,29 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint - ** PRAGMA [database.]wal_blocking_checkpoint + ** PRAGMA [database.]wal_checkpoint = passive|full|restart ** ** Checkpoint the database. */ - if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 - || sqlite3StrICmp(zLeft, "wal_blocking_checkpoint")==0 - ){ - int bBlock = (zLeft[14]!=0); + if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); - assert( bBlock==(sqlite3StrICmp(zLeft, "wal_checkpoint")!=0) ); + int eMode = SQLITE_CHECKPOINT_PASSIVE; + if( zRight ){ + if( sqlite3StrICmp(zRight, "full")==0 ){ + eMode = SQLITE_CHECKPOINT_FULL; + }else if( sqlite3StrICmp(zRight, "restart")==0 ){ + eMode = SQLITE_CHECKPOINT_RESTART; + } + } if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeAddOp2(v, OP_Checkpoint, iBt, bBlock); + sqlite3VdbeSetNumCols(v, 3); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); + + sqlite3VdbeAddOp2(v, OP_Checkpoint, iBt, eMode); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); }else /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e08d5ce596..8baf9ae029 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6145,6 +6145,89 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +/* +** +** CAPI3REF: Checkpoint a database +** +** Run a checkpoint operation on WAL database zDb attached to database +** handle db. The specific operation is determined by the value of the +** eMode parameter: +** +**
+**
SQLITE_CHECKPOINT_PASSIVE
+** Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish. Sync the db file if all frames in the log +** are checkpointed. This mode is the same as calling +** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** +**
SQLITE_CHECKPOINT_FULL
+** This mode blocks (calls the busy-handler callback) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. It then checkpoints all frames in the log file and syncs the +** database file. This call blocks database writers while it is running, +** but not database readers. +** +**
SQLITE_CHECKPOINT_RESTART
+** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after +** checkpointing the log file it blocks (calls the busy-handler callback) +** until all readers are reading from the database file only. This ensures +** that the next client to write to the database file restarts the log file +** from the beginning. This call blocks database writers while it is running, +** but not database readers. +**
+** +** If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to +** the total number of checkpointed frames (including any that were already +** checkpointed when this function is called). *pnLog and *pnCkpt may be +** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. +** If no values are available because of an error, they are both set to -1 +** before returning to communicate this to the caller. +** +** All calls obtain an exclusive "checkpoint" lock on the database file. If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive +** "writer" lock on the database file. If the writer lock cannot be obtained +** immediately, and a busy-handler is configured, it is invoked and the writer +** lock retried until either the busy-handler returns 0 or the lock is +** successfully obtained. The busy-handler is also invoked while waiting for +** database readers as described above. If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. SQLITE_BUSY is returned in this case. +** +** If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned to the caller. If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code returned to the caller immediately. If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +*/ +int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 + + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8b4fe599eb..c6e7f6e67a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3036,7 +3036,7 @@ CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); const char *sqlite3JournalModename(int); -int sqlite3Checkpoint(sqlite3*, int, int); +int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); /* Declarations for functions in fkey.c. All of these are replaced by diff --git a/src/vdbe.c b/src/vdbe.c index dcff83f17a..0e9f27d101 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5216,13 +5216,33 @@ case OP_AggFinal: { } #ifndef SQLITE_OMIT_WAL -/* Opcode: Checkpoint P1 P2 * * * +/* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. If P2 is non-zero, this is a blocking checkpoint. +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL +** or RESTART. */ case OP_Checkpoint: { - rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2); + int nLog = -1; /* Number of pages in WAL log */ + int nCkpt = -1; /* Number of checkpointed pages */ + int bBusy = 0; + assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE + || pOp->p2==SQLITE_CHECKPOINT_FULL + || pOp->p2==SQLITE_CHECKPOINT_RESTART + ); + rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &nLog, &nCkpt); + if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + bBusy = 1; + } + + aMem[1].u.i = bBusy; + aMem[2].u.i = nLog; + aMem[3].u.i = nCkpt; + MemSetTypeFlag(&aMem[1], MEM_Int); + MemSetTypeFlag(&aMem[2], MEM_Int); + MemSetTypeFlag(&aMem[3], MEM_Int); + break; }; #endif diff --git a/src/wal.c b/src/wal.c index 5ff9f227e0..0619822434 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1576,11 +1576,13 @@ static int walBusyLock( */ static int walCheckpoint( Wal *pWal, /* Wal connection */ + int eMode, /* One of PASSIVE, FULL or RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ int nBuf, /* Size of zBuf in bytes */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnCkpt /* Total frames checkpointed */ ){ int rc; /* Return code */ int szPage; /* Database page-size */ @@ -1610,6 +1612,10 @@ static int walCheckpoint( goto walcheckpoint_out; } + pInfo = walCkptInfo(pWal); + mxPage = pWal->hdr.nPage; + if( pnCkpt ) *pnCkpt = pInfo->nBackfill; + /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might ** overwrite database pages that are in use by active readers and thus @@ -1617,8 +1623,6 @@ static int walCheckpoint( */ do { mxSafeFrame = pWal->hdr.mxFrame; - mxPage = pWal->hdr.nPage; - pInfo = walCkptInfo(pWal); for(i=1; iaReadMark[i]; if( mxSafeFrame>=y ){ @@ -1634,7 +1638,8 @@ static int walCheckpoint( } } } - }while( xBusy && mxSafeFramehdr.mxFrame && xBusy(pBusyArg) ); + }while( eMode!=SQLITE_CHECKPOINT_PASSIVE + && xBusy && mxSafeFramehdr.mxFrame && xBusy(pBusyArg) ); if( pInfo->nBackfillnBackfill = mxSafeFrame; + if( pnCkpt ) *pnCkpt = mxSafeFrame; } } /* Release the reader lock held while backfilling */ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); - - if( xBusy && rc==SQLITE_OK && pWal->hdr.mxFrame==mxSafeFrame ){ - assert( pWal->writeLock ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); - if( rc==SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); - } - } } if( rc==SQLITE_BUSY ){ @@ -1706,6 +1704,17 @@ static int walCheckpoint( rc = SQLITE_OK; } + if( rc==SQLITE_OK + && eMode==SQLITE_CHECKPOINT_RESTART + && pWal->hdr.mxFrame==mxSafeFrame + ){ + assert( pWal->writeLock ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } + } + walcheckpoint_out: walIteratorFree(pIter); return rc; @@ -1737,7 +1746,9 @@ int sqlite3WalClose( if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = sqlite3WalCheckpoint(pWal, 0, 0, sync_flags, nBuf, zBuf); + rc = sqlite3WalCheckpoint( + pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 + ); if( rc==SQLITE_OK ){ isDelete = 1; } @@ -2658,11 +2669,14 @@ int sqlite3WalFrames( */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ + int eMode, /* PASSIVE, FULL or RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ int nBuf, /* Size of temporary buffer */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ ){ int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ @@ -2684,7 +2698,7 @@ int sqlite3WalCheckpoint( ** to prevent any writers from running while the checkpoint is underway. ** This has to be done before the call to walIndexReadHdr() below. */ - if( xBusy ){ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); if( rc==SQLITE_OK ) pWal->writeLock = 1; } @@ -2694,7 +2708,9 @@ int sqlite3WalCheckpoint( rc = walIndexReadHdr(pWal, &isChanged); } if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, xBusy, pBusyArg, sync_flags, nBuf, zBuf); + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + rc = walCheckpoint( + pWal, eMode, xBusy, pBusyArg, sync_flags, nBuf, zBuf, pnCkpt); } if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was diff --git a/src/wal.h b/src/wal.h index e9c614fe85..2039c701cf 100644 --- a/src/wal.h +++ b/src/wal.h @@ -20,22 +20,22 @@ #include "sqliteInt.h" #ifdef SQLITE_OMIT_WAL -# define sqlite3WalOpen(x,y,z) 0 -# define sqlite3WalClose(w,x,y,z) 0 -# define sqlite3WalBeginReadTransaction(y,z) 0 +# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalClose(w,x,y,z) 0 +# define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 -# define sqlite3WalDbsize(y) 0 -# define sqlite3WalBeginWriteTransaction(y) 0 -# define sqlite3WalEndWriteTransaction(x) 0 -# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalRead(v,w,x,y,z) 0 +# define sqlite3WalDbsize(y) 0 +# define sqlite3WalBeginWriteTransaction(y) 0 +# define sqlite3WalEndWriteTransaction(x) 0 +# define sqlite3WalUndo(x,y,z) 0 # define sqlite3WalSavepoint(y,z) -# define sqlite3WalSavepointUndo(y,z) 0 -# define sqlite3WalFrames(u,v,w,x,y,z) 0 -# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0 -# define sqlite3WalCallback(z) 0 -# define sqlite3WalExclusiveMode(y,z) 0 -# define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalSavepointUndo(y,z) 0 +# define sqlite3WalFrames(u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 +# define sqlite3WalCallback(z) 0 +# define sqlite3WalExclusiveMode(y,z) 0 +# define sqlite3WalHeapMemory(z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -86,11 +86,14 @@ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ int sqlite3WalCheckpoint( Wal *pWal, /* Write-ahead log connection */ + int eMode, /* One of PASSIVE, FULL and RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ int nBuf, /* Size of buffer nBuf */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ ); /* Return the value to pass to a sqlite3_wal_hook callback, the diff --git a/test/wal5.test b/test/wal5.test index 24d6353a62..aa2014760c 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -82,7 +82,7 @@ do_multiclient_test tn { # set ::busy_handler_script { if {$n==5} { sql2 COMMIT } } do_test 1.$tn.5 { - sql1 { PRAGMA wal_blocking_checkpoint } + sql1 { PRAGMA wal_checkpoint = RESTART } list [db_page_count] [wal_page_count] $::nBusyHandler } {6 12 6} do_test 1.$tn.6 { @@ -109,7 +109,8 @@ do_multiclient_test tn { if {$n==7} { sql3 COMMIT } } do_test 1.$tn.11 { - sql1 { PRAGMA wal_blocking_checkpoint } +breakpoint + sql1 { PRAGMA wal_checkpoint = RESTART } list [db_page_count] [wal_page_count] $::nBusyHandler } {10 5 8} do_test 1.$tn.12 { set ::db_file_size } 10 From bdd9af0fc82d956aa0a82784ebc1a232068b76fe Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Nov 2010 16:14:24 +0000 Subject: [PATCH 3/6] Update test cases to account for the change in the previous commit. FossilOrigin-Name: ae089ec881beb1c2d278303ba964aaeffad1afe4 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/attach.test | 1 + test/pager1.test | 2 +- test/wal.test | 40 +++++++++++++++++++--------------------- test/wal2.test | 8 ++++---- test/wal3.test | 6 +++--- 7 files changed, 39 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 7dd2eb66e6..d01e3580e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sinterface\sto\sthe\sblocking\swal-checkpoint\sfunctionality. -D 2010-11-18T12:11:05 +C Update\stest\scases\sto\saccount\sfor\sthe\schange\sin\sthe\sprevious\scommit. +D 2010-11-18T16:14:24 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -256,7 +256,7 @@ F test/async2.test bf5e2ca2c96763b4cba3d016249ad7259a5603b6 F test/async3.test 93edaa9122f498e56ea98c36c72abc407f4fb11e F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a F test/async5.test f3592d79c84d6e83a5f50d3fd500445f7d97dfdf -F test/attach.test ce9660e51768fab93cf129787be886c5d6c4fd81 +F test/attach.test f2b4ac6931f45695082b9f02be959c9c262e4f4d F test/attach2.test a295d2d7061adcee5884ef4a93c7c96a82765437 F test/attach3.test bd9830bc3a0d22ed1310c9bff6896927937017dc F test/attachmalloc.test 1d5b821a676f7bf0b00d87cc106b78966789ba57 @@ -570,7 +570,7 @@ F test/notify3.test d60923e186e0900f4812a845fcdfd8eea096e33a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec -F test/pager1.test 07b06b89d50bc38bb118a18b99c7bee645b315de +F test/pager1.test 1e07368795dc6205a046a7a29f4455a9d5195d19 F test/pager2.test 0fbb6b6dc40ce1fecfe758c555a748ad2e9beaa3 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 9de4d3e0c59970b4c6cb8dac511fa242f335d8a7 @@ -823,9 +823,9 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 70227190e713b3e7eb2a7d5ec3510b66db01f327 -F test/wal2.test c794b8b257af54190bb913678ad3984cbf3311b9 -F test/wal3.test 957a5f2a8fe8a6ff01de1a15285ecf2f376fcaf8 +F test/wal.test dea22218fd68c61fe206f53d508a0552894144b7 +F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7 +F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test 4e2854d7584dd97a73e7ce0f47bcbbe5c592fe29 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe @@ -887,7 +887,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 7e3fc2c833a5baa08820c499867b6902bdc2ed5a -R 1672f1722e4277222ee64bddb8543abf +P 72787c010c8944e8fcf9c98aa4482f129142d8e9 +R e8b8625daa4f884d804e2c6d9b55ed2e U dan -Z 2e080f5eeb72d6f9662d3b3c376a7b7c +Z f89794a71de2f3c0c0dca2ba94548ae2 diff --git a/manifest.uuid b/manifest.uuid index 3807e38807..109db3b96b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72787c010c8944e8fcf9c98aa4482f129142d8e9 \ No newline at end of file +ae089ec881beb1c2d278303ba964aaeffad1afe4 \ No newline at end of file diff --git a/test/attach.test b/test/attach.test index 61e1272e74..a5af8f751d 100644 --- a/test/attach.test +++ b/test/attach.test @@ -835,4 +835,5 @@ do_test attach-10.2 { PRAGMA database_list; }] 9 end } {4 noname {} 5 inmem {}} + finish_test diff --git a/test/pager1.test b/test/pager1.test index dfc49af8b5..79a04caffb 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -1992,7 +1992,7 @@ do_test pager1-22.1.1 { INSERT INTO ko DEFAULT VALUES; } execsql { PRAGMA wal_checkpoint } -} {} +} {0 -1 -1} do_test pager1-22.2.1 { testvfs tv -default 1 tv filter xSync diff --git a/test/wal.test b/test/wal.test index 1db08b5ec9..70dabda6b3 100644 --- a/test/wal.test +++ b/test/wal.test @@ -287,8 +287,8 @@ do_test wal-4.6.1 { INSERT INTO t2 VALUES('y', 'z'); ROLLBACK TO save; COMMIT; - SELECT * FROM t2; } + execsql { SELECT * FROM t2 } } {w x} @@ -542,7 +542,7 @@ do_multiclient_test tn { } {1 2 3 4 5 6 7 8 9 10} do_test wal-10.$tn.12 { catchsql { PRAGMA wal_checkpoint } - } {0 {}} ;# Reader no longer block checkpoints + } {0 {0 13 13}} ;# Reader no longer block checkpoints do_test wal-10.$tn.13 { execsql { INSERT INTO t1 VALUES(11, 12) } sql2 {SELECT * FROM t1} @@ -552,7 +552,7 @@ do_multiclient_test tn { # do_test wal-10.$tn.14 { catchsql { PRAGMA wal_checkpoint } - } {0 {}} + } {0 {0 15 13}} # The following series of test cases used to verify another blocking # case in WAL - a case which no longer blocks. @@ -562,10 +562,10 @@ do_multiclient_test tn { } {1 2 3 4 5 6 7 8 9 10 11 12} do_test wal-10.$tn.16 { catchsql { PRAGMA wal_checkpoint } - } {0 {}} + } {0 {0 15 15}} do_test wal-10.$tn.17 { execsql { PRAGMA wal_checkpoint } - } {} + } {0 15 15} do_test wal-10.$tn.18 { sql3 { BEGIN; SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12} @@ -588,13 +588,13 @@ do_multiclient_test tn { # do_test wal-10.$tn.23 { execsql { PRAGMA wal_checkpoint } - } {} + } {0 17 17} do_test wal-10.$tn.24 { sql2 { BEGIN; SELECT * FROM t1; } } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} do_test wal-10.$tn.25 { execsql { PRAGMA wal_checkpoint } - } {} + } {0 17 17} do_test wal-10.$tn.26 { catchsql { INSERT INTO t1 VALUES(15, 16) } } {0 {}} @@ -611,11 +611,11 @@ do_multiclient_test tn { do_test wal-10.$tn.29 { execsql { INSERT INTO t1 VALUES(19, 20) } catchsql { PRAGMA wal_checkpoint } - } {0 {}} + } {0 {0 6 0}} do_test wal-10.$tn.30 { code3 { sqlite3_finalize $::STMT } execsql { PRAGMA wal_checkpoint } - } {} + } {0 6 0} # At one point, if a reader failed to upgrade to a writer because it # was reading an old snapshot, the write-locks were not being released. @@ -654,7 +654,7 @@ do_multiclient_test tn { } {a b c d} do_test wal-10.$tn.36 { catchsql { PRAGMA wal_checkpoint } - } {0 {}} + } {0 {0 16 16}} do_test wal-10.$tn.36 { sql3 { INSERT INTO t1 VALUES('e', 'f') } sql2 { SELECT * FROM t1 } @@ -662,7 +662,7 @@ do_multiclient_test tn { do_test wal-10.$tn.37 { sql2 COMMIT execsql { PRAGMA wal_checkpoint } - } {} + } {0 18 18} } #------------------------------------------------------------------------- @@ -797,8 +797,8 @@ do_test wal-12.5 { UPDATE t1 SET y = 1 WHERE x = 'A'; PRAGMA wal_checkpoint; UPDATE t1 SET y = 0 WHERE x = 'A'; - SELECT * FROM t2; } + execsql { SELECT * FROM t2 } } {B 2} do_test wal-12.6 { file copy -force test.db test2.db @@ -1029,14 +1029,14 @@ catch { db close } foreach {tn ckpt_cmd ckpt_res ckpt_main ckpt_aux} { 1 {sqlite3_wal_checkpoint db} SQLITE_OK 1 1 2 {sqlite3_wal_checkpoint db ""} SQLITE_OK 1 1 - 3 {db eval "PRAGMA wal_checkpoint"} {} 1 1 + 3 {db eval "PRAGMA wal_checkpoint"} {0 16 16} 1 1 4 {sqlite3_wal_checkpoint db main} SQLITE_OK 1 0 5 {sqlite3_wal_checkpoint db aux} SQLITE_OK 0 1 6 {sqlite3_wal_checkpoint db temp} SQLITE_OK 0 0 - 7 {db eval "PRAGMA main.wal_checkpoint"} {} 1 0 - 8 {db eval "PRAGMA aux.wal_checkpoint"} {} 0 1 - 9 {db eval "PRAGMA temp.wal_checkpoint"} {} 0 0 + 7 {db eval "PRAGMA main.wal_checkpoint"} {0 10 10} 1 0 + 8 {db eval "PRAGMA aux.wal_checkpoint"} {0 16 16} 0 1 + 9 {db eval "PRAGMA temp.wal_checkpoint"} {0 -1 -1} 0 0 } { do_test wal-16.$tn.1 { file delete -force test2.db test2.db-wal test2.db-journal @@ -1400,10 +1400,8 @@ do_test wal-20.2 { } {0} do_test wal-20.3 { close $::buddy - execsql { - PRAGMA wal_checkpoint; - SELECT count(*) FROM t1; - } + execsql { PRAGMA wal_checkpoint } + execsql { SELECT count(*) FROM t1 } } {16384} do_test wal-20.4 { db close @@ -1437,8 +1435,8 @@ do_test wal-21.2 { INSERT INTO t1 SELECT randomblob(900), randomblob(900) FROM t1; ROLLBACK TO s; COMMIT; - SELECT * FROM t1; } + execsql { SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12} do_test wal-21.3 { execsql { PRAGMA integrity_check } diff --git a/test/wal2.test b/test/wal2.test index a62e9b99e6..299afa4f79 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -346,7 +346,7 @@ do_test wal2-4.1 { INSERT INTO data VALUES('need xShmOpen to see this'); PRAGMA wal_checkpoint; } -} {wal} +} {wal 0 5 5} do_test wal2-4.2 { db close testvfs tvfs -noshm 1 @@ -712,7 +712,7 @@ do_test wal2-6.5.1 { INSERT INTO t2 VALUES('I', 'II'); PRAGMA journal_mode; } -} {wal exclusive wal} +} {wal exclusive 0 3 3 wal} do_test wal2-6.5.2 { execsql { PRAGMA locking_mode = normal; @@ -723,7 +723,7 @@ do_test wal2-6.5.2 { } {normal exclusive I II III IV} do_test wal2-6.5.3 { execsql { PRAGMA wal_checkpoint } -} {} +} {0 4 4} db close proc lock_control {method filename handle spec} { @@ -807,9 +807,9 @@ do_test wal2-8.1.2 { CREATE TABLE t1(x); INSERT INTO t1 VALUES(zeroblob(8188*1020)); CREATE TABLE t2(y); + PRAGMA wal_checkpoint; } execsql { - PRAGMA wal_checkpoint; SELECT rootpage>=8192 FROM sqlite_master WHERE tbl_name = 't2'; } } {1} diff --git a/test/wal3.test b/test/wal3.test index c57132e070..84e321c27f 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -427,7 +427,7 @@ do_test wal3-6.1.2 { } {o t t f} do_test wal3-6.1.3 { execsql { PRAGMA wal_checkpoint } db2 -} {} +} {0 7 7} # At this point the log file has been fully checkpointed. However, # connection [db3] holds a lock that prevents the log from being wrapped. @@ -515,7 +515,7 @@ proc lock_callback {method file handle spec} { } do_test wal3-6.2.2 { execsql { PRAGMA wal_checkpoint } -} {} +} {0 7 7} do_test wal3-6.2.3 { set ::R } {h h l b} @@ -624,7 +624,7 @@ do_test wal3-8.1 { INSERT INTO b VALUES('Markazi'); PRAGMA wal_checkpoint; } -} {wal} +} {wal 0 9 9} do_test wal3-8.2 { execsql { SELECT * FROM b } } {Tehran Qom Markazi} From f2b8dd588d9899c9a4f3afd28ed64dce08d6705b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Nov 2010 19:28:01 +0000 Subject: [PATCH 4/6] Fixes for SQLITE_BUSY handling in blocking checkpoint code. FossilOrigin-Name: 4c663a4dcc77e00558edd94f58410605b95db33a --- manifest | 18 ++++----- manifest.uuid | 2 +- src/main.c | 9 ++++- src/wal.c | 106 ++++++++++++++++++++++++++++++------------------- test/wal.test | 3 +- test/wal5.test | 89 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 172 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index de24bf1209..4cb48f95f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\swith\slatest\strunk\sfix. -D 2010-11-18T16:59:24 +C Fixes\sfor\sSQLITE_BUSY\shandling\sin\sblocking\scheckpoint\scode. +D 2010-11-18T19:28:02 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -141,7 +141,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e -F src/main.c 91465f2658911ddb51be89e7b8ee01af8584308f +F src/main.c 5927ac9ca30c46df68148ad68227329d1be41a21 F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -236,7 +236,7 @@ F src/vdbeblob.c e0ce3c54cc0c183af2ec67b63a289acf92251df4 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30 -F src/wal.c 8eca619a28a70a667c913e5927131250836377a2 +F src/wal.c 23facfd0f148ac72729fe28bbf973fe0458757b6 F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c fa22d45b2577c77146f2e894d58011d472d64103 @@ -824,11 +824,11 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test dea22218fd68c61fe206f53d508a0552894144b7 +F test/wal.test f060cae4b2164c4375109a8f803873187234661d F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test e5330471fc284d9ae62a2a18c9dfd10b6605d8b6 +F test/wal5.test 79963972107e4cabda4c4b44a64e89c3e9af463d F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -888,7 +888,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P e376480f0855c598c91529abacbd73e31d9f4713 0a95589f2166f9ce420e647b73e8c797fe8f4833 -R 87db3bd5bf4a3735ef4309e5d5955901 +P a8910e89dee326d7788b29e77820eb1e114739ca +R a15fbbfd076e96cb91858a5a2944b7af U dan -Z 9fccaf5d4045282a697f1275247ad8b5 +Z 77692281a1477b941972542152e8834d diff --git a/manifest.uuid b/manifest.uuid index b70e7c96b6..81ad5c3317 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8910e89dee326d7788b29e77820eb1e114739ca \ No newline at end of file +4c663a4dcc77e00558edd94f58410605b95db33a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4333a5873b..0870a6bcee 100644 --- a/src/main.c +++ b/src/main.c @@ -1413,16 +1413,23 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } } } - return rc; + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; } #endif /* SQLITE_OMIT_WAL */ diff --git a/src/wal.c b/src/wal.c index 0619822434..2cb0e415ee 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1543,6 +1543,14 @@ static int walBusyLock( return rc; } +/* +** The cache of the wal-index header must be valid to call this function. +** Return the page-size in bytes used by the database. +*/ +static int walPagesize(Wal *pWal){ + return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1577,10 +1585,9 @@ static int walBusyLock( static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusy)(void*), /* Function to call when busy */ + int (*xBusyCall)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ - int nBuf, /* Size of zBuf in bytes */ u8 *zBuf, /* Temporary buffer to use */ int *pnCkpt /* Total frames checkpointed */ ){ @@ -1593,11 +1600,11 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ - szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); - if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); @@ -1606,40 +1613,33 @@ static int walCheckpoint( } assert( pIter ); - /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( szPage!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - goto walcheckpoint_out; - } - pInfo = walCkptInfo(pWal); mxPage = pWal->hdr.nPage; if( pnCkpt ) *pnCkpt = pInfo->nBackfill; + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might ** overwrite database pages that are in use by active readers and thus ** cannot be backfilled from the WAL. */ - do { - mxSafeFrame = pWal->hdr.mxFrame; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>=y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = READMARK_NOT_USED; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - }else{ - goto walcheckpoint_out; - } + mxSafeFrame = pWal->hdr.mxFrame; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = READMARK_NOT_USED; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; } } - }while( eMode!=SQLITE_CHECKPOINT_PASSIVE - && xBusy && mxSafeFramehdr.mxFrame && xBusy(pBusyArg) ); + } if( pInfo->nBackfillhdr.mxFrame==mxSafeFrame - ){ + /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal + ** file has been copied into the database file, then block until all + ** readers have finished using the wal file. This ensures that the next + ** process to write to the database restarts the wal file. + */ + if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); - if( rc==SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; + }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + assert( mxSafeFrame==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } } } @@ -2680,6 +2687,7 @@ int sqlite3WalCheckpoint( ){ int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ + int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); @@ -2697,21 +2705,37 @@ int sqlite3WalCheckpoint( /* If this is a blocking-checkpoint, then obtain the write-lock as well ** to prevent any writers from running while the checkpoint is underway. ** This has to be done before the call to walIndexReadHdr() below. + ** + ** If the writer lock cannot be obtained, then a passive checkpoint is + ** run instead. Since the checkpointer is not holding the writer lock, + ** there is no point in blocking waiting for any readers. Assuming no + ** other error occurs, this function will return SQLITE_BUSY to the caller. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ) pWal->writeLock = 1; + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + rc = SQLITE_OK; + } } - /* Copy data from the log to the database file. */ + /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); } - if( rc==SQLITE_OK ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - rc = walCheckpoint( - pWal, eMode, xBusy, pBusyArg, sync_flags, nBuf, zBuf, pnCkpt); + + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK && pWal->hdr.mxFrame ){ + if( walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags,zBuf,pnCkpt); + } } + if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was ** performed, then the pager-cache associated with pWal is now @@ -2727,7 +2751,7 @@ int sqlite3WalCheckpoint( walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); - return rc; + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } /* Return the value to pass to a sqlite3_wal_hook callback, the diff --git a/test/wal.test b/test/wal.test index 70dabda6b3..339661e2ee 100644 --- a/test/wal.test +++ b/test/wal.test @@ -845,6 +845,7 @@ do_test wal-13.1.2 { sqlite3 db test.db execsql { SELECT * FROM t2 } } {B 2} +breakpoint do_test wal-13.1.3 { db close file exists test.db-wal @@ -1029,7 +1030,7 @@ catch { db close } foreach {tn ckpt_cmd ckpt_res ckpt_main ckpt_aux} { 1 {sqlite3_wal_checkpoint db} SQLITE_OK 1 1 2 {sqlite3_wal_checkpoint db ""} SQLITE_OK 1 1 - 3 {db eval "PRAGMA wal_checkpoint"} {0 16 16} 1 1 + 3 {db eval "PRAGMA wal_checkpoint"} {0 10 10} 1 1 4 {sqlite3_wal_checkpoint db main} SQLITE_OK 1 0 5 {sqlite3_wal_checkpoint db aux} SQLITE_OK 0 1 diff --git a/test/wal5.test b/test/wal5.test index 49cd839761..ffcf044866 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -21,10 +21,12 @@ ifcapable !wal {finish_test ; return } set testprefix wal5 +proc db_page_count {{file test.db}} { expr [file size $file] / 1024 } +proc wal_page_count {{file test.db}} { wal_frame_count ${file}-wal 1024 } + + do_multiclient_test tn { - proc db_page_count {} { expr [file size test.db] / 1024 } - proc wal_page_count {} { wal_frame_count test.db-wal 1024 } set ::nBusyHandler 0 set ::busy_handler_script "" @@ -115,4 +117,87 @@ do_multiclient_test tn { do_test 1.$tn.12 { set ::db_file_size } 10 } + +#------------------------------------------------------------------------- +# This block of tests explores checkpoint operations on more than one +# database file. +# +proc setup_and_attach_aux {} { + sql1 { ATTACH 'test.db2' AS aux } + sql2 { ATTACH 'test.db2' AS aux } + sql3 { ATTACH 'test.db2' AS aux } + sql1 { + PRAGMA main.page_size=1024; PRAGMA main.journal_mode=WAL; + PRAGMA aux.page_size=1024; PRAGMA aux.journal_mode=WAL; + } +} + +proc file_page_counts {} { + list [db_page_count test.db ] \ + [wal_page_count test.db ] \ + [db_page_count test.db2] \ + [wal_page_count test.db2] +} + +do_multiclient_test tn { + setup_and_attach_aux + do_test 2.1.$tn.1 { + sql1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + } + } {} + + do_test 2.2.$tn.2 { file_page_counts } {1 5 1 5} + do_test 2.1.$tn.3 { sql1 { PRAGMA wal_checkpoint } } {0 5 5} + do_test 2.1.$tn.4 { file_page_counts } {2 5 2 5} +} + +do_multiclient_test tn { + + setup_and_attach_aux + + do_test 2.2.$tn.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + INSERT INTO t2 VALUES(3, 4); + } + } {} + + do_test 2.2.$tn.2 { file_page_counts } {1 5 1 7} + do_test 2.2.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} + do_test 2.2.$tn.4 { sql1 { PRAGMA wal_checkpoint = RESTART } } {1 5 5} + do_test 2.2.$tn.5 { file_page_counts } {2 5 2 7} +} + +do_multiclient_test tn { + + setup_and_attach_aux + + do_test 2.3.$tn.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + } + } {} + + do_test 2.3.$tn.2 { file_page_counts } {1 5 1 5} + do_test 2.3.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} + do_test 2.3.$tn.4 { sql1 { INSERT INTO t1 VALUES(3, 4) } } {} + do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} + do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} + + do_test 2.3.$tn.7 { sql1 { PRAGMA wal_checkpoint = FULL } } {1 7 5} + do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} +} + + + finish_test From 08756372d0ef318407d9ca9b3439c1ae7cf1f71b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 19 Nov 2010 07:17:09 +0000 Subject: [PATCH 5/6] Add extra test cases for blocking checkpoints. FossilOrigin-Name: ac348ae25cb0239dc525bb473cc83cdb1b3de205 --- manifest | 12 ++++----- manifest.uuid | 2 +- test/wal5.test | 72 ++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 4cb48f95f4..6055c0cab3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sSQLITE_BUSY\shandling\sin\sblocking\scheckpoint\scode. -D 2010-11-18T19:28:02 +C Add\sextra\stest\scases\sfor\sblocking\scheckpoints. +D 2010-11-19T07:17:10 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -828,7 +828,7 @@ F test/wal.test f060cae4b2164c4375109a8f803873187234661d F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test 79963972107e4cabda4c4b44a64e89c3e9af463d +F test/wal5.test b467d39f88ce0e814a3cf381f0f6a014664d2e85 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -888,7 +888,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a8910e89dee326d7788b29e77820eb1e114739ca -R a15fbbfd076e96cb91858a5a2944b7af +P 4c663a4dcc77e00558edd94f58410605b95db33a +R d43362578e7138fb690b169b023fbca4 U dan -Z 77692281a1477b941972542152e8834d +Z c818aa76c45599e0e0a501a870acd353 diff --git a/manifest.uuid b/manifest.uuid index 81ad5c3317..4c59d4499f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c663a4dcc77e00558edd94f58410605b95db33a \ No newline at end of file +ac348ae25cb0239dc525bb473cc83cdb1b3de205 \ No newline at end of file diff --git a/test/wal5.test b/test/wal5.test index ffcf044866..c2fbfdd51b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -139,6 +139,9 @@ proc file_page_counts {} { [wal_page_count test.db2] } +# Test that executing "PRAGMA wal_checkpoint" checkpoints all attached +# databases, not just the main db. +# do_multiclient_test tn { setup_and_attach_aux do_test 2.1.$tn.1 { @@ -149,16 +152,13 @@ do_multiclient_test tn { INSERT INTO t2 VALUES(1, 2); } } {} - do_test 2.2.$tn.2 { file_page_counts } {1 5 1 5} do_test 2.1.$tn.3 { sql1 { PRAGMA wal_checkpoint } } {0 5 5} do_test 2.1.$tn.4 { file_page_counts } {2 5 2 5} } do_multiclient_test tn { - setup_and_attach_aux - do_test 2.2.$tn.1 { execsql { CREATE TABLE t1(a, b); @@ -168,7 +168,6 @@ do_multiclient_test tn { INSERT INTO t2 VALUES(3, 4); } } {} - do_test 2.2.$tn.2 { file_page_counts } {1 5 1 7} do_test 2.2.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} do_test 2.2.$tn.4 { sql1 { PRAGMA wal_checkpoint = RESTART } } {1 5 5} @@ -176,9 +175,7 @@ do_multiclient_test tn { } do_multiclient_test tn { - setup_and_attach_aux - do_test 2.3.$tn.1 { execsql { CREATE TABLE t1(a, b); @@ -187,17 +184,76 @@ do_multiclient_test tn { INSERT INTO t2 VALUES(1, 2); } } {} - do_test 2.3.$tn.2 { file_page_counts } {1 5 1 5} do_test 2.3.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} do_test 2.3.$tn.4 { sql1 { INSERT INTO t1 VALUES(3, 4) } } {} do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} - do_test 2.3.$tn.7 { sql1 { PRAGMA wal_checkpoint = FULL } } {1 7 5} do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} } +# Check that checkpoints block on the correct locks. And respond correctly +# if they cannot obtain those locks. There are three locks that a checkpoint +# may block on (in the following order): +# +# 1. The writer lock: FULL and RESTART checkpoints block until any writer +# process releases its lock. +# +# 2. Readers using part of the log file. FULL and RESTART checkpoints block +# until readers using part (but not all) of the log file have finished. +# +# 3. Readers using any of the log file. After copying data into the +# database file, RESTART checkpoints block until readers using any part +# of the log file have finished. +# +foreach {tn1 checkpoint busy_on ckpt_expected expected} { + 1 PASSIVE - {0 5 5} - + 2 TYPO - {0 5 5} - + + 3 FULL - {0 7 7} 2 + 4 FULL 1 {1 5 5} 1 + 5 FULL 2 {1 7 5} 2 + 6 FULL 3 {0 7 7} 2 + + 7 RESTART - {0 7 7} 3 + 8 RESTART 1 {1 5 5} 1 + 9 RESTART 2 {1 7 5} 2 + 10 RESTART 3 {1 7 7} 3 + +} { + do_multiclient_test tn { + setup_and_attach_aux + + proc busyhandler {x} { + set ::max_busyhandler $x + if {$::busy_on!="-" && $x==$::busy_on} { return 1 } + switch -- $x { + 1 { sql2 "COMMIT ; BEGIN ; SELECT * FROM t1" } + 2 { sql3 "COMMIT" } + 3 { sql2 "COMMIT" } + } + return 0 + } + set ::max_busyhandler - + + do_test 2.4.$tn1.$tn.1 { + sql1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + sql2 { BEGIN; INSERT INTO t1 VALUES(3, 4) } + sql3 { BEGIN; SELECT * FROM t1 } + } {1 2} + + do_test 2.4.$tn1.$tn.2 { + code1 { db busy busyhandler } + sql1 "PRAGMA wal_checkpoint = $checkpoint" + } $ckpt_expected + do_test 2.4.$tn1.$tn.3 { set ::max_busyhandler } $expected + } +} finish_test + From 24cd616ed8b7f32466a04f9099708feeabd34d52 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 19 Nov 2010 09:58:11 +0000 Subject: [PATCH 6/6] Add file test/tt3_checkpoint.c that adds a multi-threaded test for blocking checkpoints to threadtest3. FossilOrigin-Name: 648dd157ef3b7b790764698fd4dd7107c25212c9 --- main.mk | 4 +- manifest | 17 ++--- manifest.uuid | 2 +- test/threadtest3.c | 6 +- test/tt3_checkpoint.c | 150 ++++++++++++++++++++++++++++++++++++++++++ test/wal5.test | 3 + 6 files changed, 170 insertions(+), 12 deletions(-) create mode 100644 test/tt3_checkpoint.c diff --git a/main.mk b/main.mk index 032e67e423..edcb977fc9 100644 --- a/main.mk +++ b/main.mk @@ -530,8 +530,8 @@ test: testfixture$(EXE) sqlite3$(EXE) # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # -threadtest3$(EXE): sqlite3.c $(TOP)/test/threadtest3.c - $(TCCX) -O2 sqlite3.c $(TOP)/test/threadtest3.c \ +threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c + $(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \ -o threadtest3$(EXE) $(THREADLIB) threadtest: threadtest3$(EXE) diff --git a/manifest b/manifest index 6055c0cab3..40abf7203b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stest\scases\sfor\sblocking\scheckpoints. -D 2010-11-19T07:17:10 +C Add\sfile\stest/tt3_checkpoint.c\sthat\sadds\sa\smulti-threaded\stest\sfor\sblocking\scheckpoints\sto\sthreadtest3. +D 2010-11-19T09:58:11 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -99,7 +99,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 497c8cb6ae132c88fa184e5e454b0e6336da5693 +F main.mk 2c360958d2981a394ce021a9dfc842577f7935df F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -668,7 +668,7 @@ F test/thread2.test e08034b83fe9693ade77049732518e5b3d2d700d F test/thread_common.tcl 2aa6f2fdcd4d6e461169c3e5ca098eebf643b863 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c d6d209190c7110f9a7e6a8154bdc3260efdbf8b7 +F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 @@ -794,6 +794,7 @@ F test/triggerA.test eaf11a29db2a11967d2d4b49d37f92bce598194e F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 2a23edcc00684d084902ba5ec93e721775c3a70a F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7 +F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test a0f66bf12f80fad89493535474f7a6d16fa58150 @@ -828,7 +829,7 @@ F test/wal.test f060cae4b2164c4375109a8f803873187234661d F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test b467d39f88ce0e814a3cf381f0f6a014664d2e85 +F test/wal5.test 1f99651d856c8b9e1376781c981d1b903e93a478 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -888,7 +889,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 4c663a4dcc77e00558edd94f58410605b95db33a -R d43362578e7138fb690b169b023fbca4 +P ac348ae25cb0239dc525bb473cc83cdb1b3de205 +R d259a6023fb0ad6db4b616b7c9bd726d U dan -Z c818aa76c45599e0e0a501a870acd353 +Z 699d2f3824e67293f11a7075160f739b diff --git a/manifest.uuid b/manifest.uuid index 4c59d4499f..ee5c3341d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac348ae25cb0239dc525bb473cc83cdb1b3de205 \ No newline at end of file +648dd157ef3b7b790764698fd4dd7107c25212c9 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index 82b4708d85..cb7e2fa41b 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1394,6 +1394,7 @@ static void dynamic_triggers(int nMs){ print_and_free_err(&err); } +#include "tt3_checkpoint.c" int main(int argc, char **argv){ struct ThreadTest { @@ -1408,8 +1409,11 @@ int main(int argc, char **argv){ { walthread5, "walthread5", 1000 }, { walthread5, "walthread5", 1000 }, - { cgt_pager_1, "cgt_pager_1", 0 }, + { cgt_pager_1, "cgt_pager_1", 0 }, { dynamic_triggers, "dynamic_triggers", 20000 }, + + { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 }, + { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 }, }; int i; diff --git a/test/tt3_checkpoint.c b/test/tt3_checkpoint.c new file mode 100644 index 0000000000..3c28f0d3a5 --- /dev/null +++ b/test/tt3_checkpoint.c @@ -0,0 +1,150 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the test program "threadtest3". Despite being a C +** file it is not compiled separately, but included by threadtest3.c using +** the #include directive normally used with header files. +** +** This file contains the implementation of test cases: +** +** checkpoint_starvation_1 +** checkpoint_starvation_2 +*/ + +/* +** Both test cases involve 1 writer/checkpointer thread and N reader threads. +** +** Each reader thread performs a series of read transactions, one after +** another. Each read transaction lasts for 100 ms. +** +** The writer writes transactions as fast as possible. It uses a callback +** registered with sqlite3_wal_hook() to try to keep the WAL-size limited to +** around 50 pages. +** +** In test case checkpoint_starvation_1, the auto-checkpoint uses +** SQLITE_CHECKPOINT_PASSIVE. In checkpoint_starvation_2, it uses RESTART. +** The expectation is that in the first case the WAL file will grow very +** large, and in the second will be limited to the 50 pages or thereabouts. +** However, the overall transaction throughput will be lower for +** checkpoint_starvation_2, as every checkpoint will block for up to 200 ms +** waiting for readers to clear. +*/ + +/* Frame limit used by the WAL hook for these tests. */ +#define CHECKPOINT_STARVATION_FRAMELIMIT 50 + +/* Duration in ms of each read transaction */ +#define CHECKPOINT_STARVATION_READMS 100 + +struct CheckpointStarvationCtx { + int eMode; + int nMaxFrame; +}; +typedef struct CheckpointStarvationCtx CheckpointStarvationCtx; + +static int checkpoint_starvation_walhook( + void *pCtx, + sqlite3 *db, + const char *zDb, + int nFrame +){ + CheckpointStarvationCtx *p = (CheckpointStarvationCtx *)pCtx; + if( nFrame>p->nMaxFrame ){ + p->nMaxFrame = nFrame; + } + if( nFrame>=CHECKPOINT_STARVATION_FRAMELIMIT ){ + sqlite3_wal_checkpoint_v2(db, zDb, p->eMode, 0, 0); + } + return SQLITE_OK; +} + +static char *checkpoint_starvation_reader(int iTid, int iArg){ + Error err = {0}; + Sqlite db = {0}; + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + i64 iCount1, iCount2; + sql_script(&err, &db, "BEGIN"); + iCount1 = execsql_i64(&err, &db, "SELECT count(x) FROM t1"); + usleep(CHECKPOINT_STARVATION_READMS*1000); + iCount2 = execsql_i64(&err, &db, "SELECT count(x) FROM t1"); + sql_script(&err, &db, "COMMIT"); + + if( iCount1!=iCount2 ){ + test_error(&err, "Isolation failure - %lld %lld", iCount1, iCount2); + } + } + closedb(&err, &db); + + print_and_free_err(&err); + return 0; +} + +static void checkpoint_starvation_main(int nMs, CheckpointStarvationCtx *p){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + int nInsert = 0; + int i; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "PRAGMA page_size = 1024;" + "PRAGMA journal_mode = WAL;" + "CREATE TABLE t1(x);" + ); + + setstoptime(&err, nMs); + + for(i=0; i<4; i++){ + launch_thread(&err, &threads, checkpoint_starvation_reader, 0); + usleep(CHECKPOINT_STARVATION_READMS*1000/4); + } + + sqlite3_wal_hook(db.db, checkpoint_starvation_walhook, (void *)p); + while( !timetostop(&err) ){ + sql_script(&err, &db, "INSERT INTO t1 VALUES(randomblob(1200))"); + nInsert++; + } + + printf(" Checkpoint mode : %s\n", + p->eMode==SQLITE_CHECKPOINT_PASSIVE ? "PASSIVE" : "RESTART" + ); + printf(" Peak WAL : %d frames\n", p->nMaxFrame); + printf(" Transaction count: %d transactions\n", nInsert); + + join_all_threads(&err, &threads); + closedb(&err, &db); + print_and_free_err(&err); +} + +static void checkpoint_starvation_1(int nMs){ + Error err = {0}; + CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_PASSIVE, 0 }; + checkpoint_starvation_main(nMs, &ctx); + if( ctx.nMaxFrame<(CHECKPOINT_STARVATION_FRAMELIMIT*10) ){ + test_error(&err, "WAL failed to grow - %d frames", ctx.nMaxFrame); + } + print_and_free_err(&err); +} + +static void checkpoint_starvation_2(int nMs){ + Error err = {0}; + CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_RESTART, 0 }; + checkpoint_starvation_main(nMs, &ctx); + if( ctx.nMaxFrame>CHECKPOINT_STARVATION_FRAMELIMIT+10 ){ + test_error(&err, "WAL grew too large - %d frames", ctx.nMaxFrame); + } + print_and_free_err(&err); +} + + diff --git a/test/wal5.test b/test/wal5.test index c2fbfdd51b..2ea4805795 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -207,6 +207,9 @@ do_multiclient_test tn { # database file, RESTART checkpoints block until readers using any part # of the log file have finished. # +# This test case involves running a checkpoint while there exist other +# processes holding all three types of locks. +# foreach {tn1 checkpoint busy_on ckpt_expected expected} { 1 PASSIVE - {0 5 5} - 2 TYPO - {0 5 5} -