Fixes for SQLITE_BUSY handling in blocking checkpoint code.

FossilOrigin-Name: 4c663a4dcc77e00558edd94f58410605b95db33a

manifest

@@ -1,5 +1,5 @@
-C Merge\swith\slatest\strunk\sfix.
+C Fixes\sfor\sSQLITE_BUSY\shandling\sin\sblocking\scheckpoint\scode.
-D 2010-11-18T16:59:24
+D 2010-11-18T19:28:02
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -141,7 +141,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
 F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e
-F src/main.c 91465f2658911ddb51be89e7b8ee01af8584308f
+F src/main.c 5927ac9ca30c46df68148ad68227329d1be41a21
 F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206
@@ -236,7 +236,7 @@ F src/vdbeblob.c e0ce3c54cc0c183af2ec67b63a289acf92251df4
 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9
 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
 F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30
-F src/wal.c 8eca619a28a70a667c913e5927131250836377a2
+F src/wal.c 23facfd0f148ac72729fe28bbf973fe0458757b6
 F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c fa22d45b2577c77146f2e894d58011d472d64103
@@ -824,11 +824,11 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
-F test/wal.test dea22218fd68c61fe206f53d508a0552894144b7
+F test/wal.test f060cae4b2164c4375109a8f803873187234661d
 F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7
 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4
 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30
-F test/wal5.test e5330471fc284d9ae62a2a18c9dfd10b6605d8b6
+F test/wal5.test 79963972107e4cabda4c4b44a64e89c3e9af463d
 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
 F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4
 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0
@@ -888,7 +888,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P e376480f0855c598c91529abacbd73e31d9f4713 0a95589f2166f9ce420e647b73e8c797fe8f4833
+P a8910e89dee326d7788b29e77820eb1e114739ca
-R 87db3bd5bf4a3735ef4309e5d5955901
+R a15fbbfd076e96cb91858a5a2944b7af
 U dan
-Z 9fccaf5d4045282a697f1275247ad8b5
+Z 77692281a1477b941972542152e8834d

manifest.uuid

@@ -1 +1 @@
-a8910e89dee326d7788b29e77820eb1e114739ca
+4c663a4dcc77e00558edd94f58410605b95db33a

src/main.c

@@ -1413,16 +1413,23 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
 int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* Used to iterate through attached dbs */
+  int bBusy = 0;                  /* True if SQLITE_BUSY has been encountered */
 
   assert( sqlite3_mutex_held(db->mutex) );
 
   for(i=0; i<db->nDb && rc==SQLITE_OK; i++){
     if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){
       rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt);
+      pnLog = 0;
+      pnCkpt = 0;
+      if( rc==SQLITE_BUSY ){
+        bBusy = 1;
+        rc = SQLITE_OK;
+      }
     }
   }
 
-  return rc;
+  return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc;
 }
 #endif /* SQLITE_OMIT_WAL */
 

src/wal.c

@@ -1543,6 +1543,14 @@ static int walBusyLock(
   return rc;
 }
 
+/*
+** The cache of the wal-index header must be valid to call this function.
+** Return the page-size in bytes used by the database.
+*/
+static int walPagesize(Wal *pWal){
+  return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
+}
+
 /*
 ** Copy as much content as we can from the WAL back into the database file
 ** in response to an sqlite3_wal_checkpoint() request or the equivalent.
@@ -1577,10 +1585,9 @@ static int walBusyLock(
 static int walCheckpoint(
   Wal *pWal,                      /* Wal connection */
   int eMode,                      /* One of PASSIVE, FULL or RESTART */
-  int (*xBusy)(void*),            /* Function to call when busy */
+  int (*xBusyCall)(void*),        /* Function to call when busy */
   void *pBusyArg,                 /* Context argument for xBusyHandler */
   int sync_flags,                 /* Flags for OsSync() (or 0) */
-  int nBuf,                       /* Size of zBuf in bytes */
   u8 *zBuf,                       /* Temporary buffer to use */
   int *pnCkpt                     /* Total frames checkpointed */
 ){
@@ -1593,11 +1600,11 @@ static int walCheckpoint(
   u32 mxPage;                     /* Max database page to write */
   int i;                          /* Loop counter */
   volatile WalCkptInfo *pInfo;    /* The checkpoint status information */
+  int (*xBusy)(void*) = 0;        /* Function to call when waiting for locks */
 
-  szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
+  szPage = walPagesize(pWal);
   testcase( szPage<=32768 );
   testcase( szPage>=65536 );
-  if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;
 
   /* Allocate the iterator */
   rc = walIteratorInit(pWal, &pIter);
@@ -1606,40 +1613,33 @@ static int walCheckpoint(
   }
   assert( pIter );
 
-  /*** TODO:  Move this test out to the caller.  Make it an assert() here ***/
-  if( szPage!=nBuf ){
-    rc = SQLITE_CORRUPT_BKPT;
-    goto walcheckpoint_out;
-  }
-
   pInfo = walCkptInfo(pWal);
   mxPage = pWal->hdr.nPage;
   if( pnCkpt ) *pnCkpt = pInfo->nBackfill;
+  if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall;
 
   /* Compute in mxSafeFrame the index of the last frame of the WAL that is
   ** safe to write into the database.  Frames beyond mxSafeFrame might
   ** overwrite database pages that are in use by active readers and thus
   ** cannot be backfilled from the WAL.
   */
-  do {
+  mxSafeFrame = pWal->hdr.mxFrame;
-    mxSafeFrame = pWal->hdr.mxFrame;
+  for(i=1; i<WAL_NREADER; i++){
-    for(i=1; i<WAL_NREADER; i++){
+    u32 y = pInfo->aReadMark[i];
-      u32 y = pInfo->aReadMark[i];
+    if( mxSafeFrame>y ){
-      if( mxSafeFrame>=y ){
+      assert( y<=pWal->hdr.mxFrame );
-        assert( y<=pWal->hdr.mxFrame );
+      rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
-        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+      if( rc==SQLITE_OK ){
-        if( rc==SQLITE_OK ){
+        pInfo->aReadMark[i] = READMARK_NOT_USED;
-          pInfo->aReadMark[i] = READMARK_NOT_USED;
+        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+      }else if( rc==SQLITE_BUSY ){
-        }else if( rc==SQLITE_BUSY ){
+        mxSafeFrame = y;
-          mxSafeFrame = y;
+        xBusy = 0;
-        }else{
+      }else{
-          goto walcheckpoint_out;
+        goto walcheckpoint_out;
-        }
       }
     }
-  }while( eMode!=SQLITE_CHECKPOINT_PASSIVE 
+  }
-       && xBusy && mxSafeFrame<pWal->hdr.mxFrame && xBusy(pBusyArg) );
 
   if( pInfo->nBackfill<mxSafeFrame
    && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))==SQLITE_OK
@@ -1704,14 +1704,21 @@ static int walCheckpoint(
     rc = SQLITE_OK;
   }
 
-  if( rc==SQLITE_OK 
+  /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal
-   && eMode==SQLITE_CHECKPOINT_RESTART 
+  ** file has been copied into the database file, then block until all
-   && pWal->hdr.mxFrame==mxSafeFrame 
+  ** readers have finished using the wal file. This ensures that the next
-  ){
+  ** process to write to the database restarts the wal file.
+  */
+  if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
     assert( pWal->writeLock );
-    rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
+    if( pInfo->nBackfill<pWal->hdr.mxFrame ){
-    if( rc==SQLITE_OK ){
+      rc = SQLITE_BUSY;
-      walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+    }else if( eMode==SQLITE_CHECKPOINT_RESTART ){
+      assert( mxSafeFrame==pWal->hdr.mxFrame );
+      rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
+      if( rc==SQLITE_OK ){
+        walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+      }
     }
   }
 
@@ -2680,6 +2687,7 @@ int sqlite3WalCheckpoint(
 ){
   int rc;                         /* Return code */
   int isChanged = 0;              /* True if a new wal-index header is loaded */
+  int eMode2 = eMode;             /* Mode to pass to walCheckpoint() */
 
   assert( pWal->ckptLock==0 );
   assert( pWal->writeLock==0 );
@@ -2697,21 +2705,37 @@ int sqlite3WalCheckpoint(
   /* If this is a blocking-checkpoint, then obtain the write-lock as well
   ** to prevent any writers from running while the checkpoint is underway.
   ** This has to be done before the call to walIndexReadHdr() below.
+  **
+  ** If the writer lock cannot be obtained, then a passive checkpoint is
+  ** run instead. Since the checkpointer is not holding the writer lock,
+  ** there is no point in blocking waiting for any readers. Assuming no 
+  ** other error occurs, this function will return SQLITE_BUSY to the caller.
   */
   if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){
     rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1);
-    if( rc==SQLITE_OK ) pWal->writeLock = 1;
+    if( rc==SQLITE_OK ){
+      pWal->writeLock = 1;
+    }else if( rc==SQLITE_BUSY ){
+      eMode2 = SQLITE_CHECKPOINT_PASSIVE;
+      rc = SQLITE_OK;
+    }
   }
 
-  /* Copy data from the log to the database file. */
+  /* Read the wal-index header. */
   if( rc==SQLITE_OK ){
     rc = walIndexReadHdr(pWal, &isChanged);
   }
-  if( rc==SQLITE_OK ){
+
-    if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
+  /* Copy data from the log to the database file. */
-    rc = walCheckpoint(
+  if( rc==SQLITE_OK && pWal->hdr.mxFrame ){
-        pWal, eMode, xBusy, pBusyArg, sync_flags, nBuf, zBuf, pnCkpt);
+    if( walPagesize(pWal)!=nBuf ){
+      rc = SQLITE_CORRUPT_BKPT;
+    }else{
+      if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
+      rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags,zBuf,pnCkpt);
+    }
   }
+
   if( isChanged ){
     /* If a new wal-index header was loaded before the checkpoint was 
     ** performed, then the pager-cache associated with pWal is now
@@ -2727,7 +2751,7 @@ int sqlite3WalCheckpoint(
   walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
   pWal->ckptLock = 0;
   WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok"));
-  return rc;
+  return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc);
 }
 
 /* Return the value to pass to a sqlite3_wal_hook callback, the

test/wal.test

@@ -845,6 +845,7 @@ do_test wal-13.1.2 {
   sqlite3 db test.db
   execsql { SELECT * FROM t2 }
 } {B 2}
+breakpoint
 do_test wal-13.1.3 {
   db close
   file exists test.db-wal
@@ -1029,7 +1030,7 @@ catch { db close }
 foreach {tn ckpt_cmd ckpt_res ckpt_main ckpt_aux} {
   1 {sqlite3_wal_checkpoint db}              SQLITE_OK     1 1
   2 {sqlite3_wal_checkpoint db ""}           SQLITE_OK     1 1
-  3 {db eval "PRAGMA wal_checkpoint"}        {0 16 16}     1 1
+  3 {db eval "PRAGMA wal_checkpoint"}        {0 10 10}     1 1
 
   4 {sqlite3_wal_checkpoint db main}         SQLITE_OK     1 0
   5 {sqlite3_wal_checkpoint db aux}          SQLITE_OK     0 1

test/wal5.test

@@ -21,10 +21,12 @@ ifcapable !wal {finish_test ; return }
 
 set testprefix wal5
 
+proc db_page_count  {{file test.db}} { expr [file size $file] / 1024 }
+proc wal_page_count {{file test.db}} { wal_frame_count ${file}-wal 1024 }
+
+
 do_multiclient_test tn {
 
-  proc db_page_count  {} { expr [file size test.db] / 1024 }
-  proc wal_page_count {} { wal_frame_count test.db-wal 1024 }
 
   set ::nBusyHandler 0
   set ::busy_handler_script ""
@@ -115,4 +117,87 @@ do_multiclient_test tn {
   do_test 1.$tn.12 { set ::db_file_size } 10
 }
 
+
+#-------------------------------------------------------------------------
+# This block of tests explores checkpoint operations on more than one 
+# database file.
+#
+proc setup_and_attach_aux {} {
+  sql1 { ATTACH 'test.db2' AS aux }
+  sql2 { ATTACH 'test.db2' AS aux }
+  sql3 { ATTACH 'test.db2' AS aux }
+  sql1 {
+    PRAGMA main.page_size=1024; PRAGMA main.journal_mode=WAL;
+    PRAGMA aux.page_size=1024;  PRAGMA aux.journal_mode=WAL;
+  }
+}
+
+proc file_page_counts {} {
+  list [db_page_count  test.db ] \
+       [wal_page_count test.db ] \
+       [db_page_count  test.db2] \
+       [wal_page_count test.db2]
+}
+
+do_multiclient_test tn {
+  setup_and_attach_aux
+  do_test 2.1.$tn.1 {
+    sql1 {
+      CREATE TABLE t1(a, b);
+      INSERT INTO t1 VALUES(1, 2);
+      CREATE TABLE aux.t2(a, b);
+      INSERT INTO t2 VALUES(1, 2);
+    }
+  } {}
+
+  do_test 2.2.$tn.2 { file_page_counts } {1 5 1 5}
+  do_test 2.1.$tn.3 { sql1 { PRAGMA wal_checkpoint } } {0 5 5}
+  do_test 2.1.$tn.4 { file_page_counts } {2 5 2 5}
+}
+
+do_multiclient_test tn {
+
+  setup_and_attach_aux
+
+  do_test 2.2.$tn.1 {
+    execsql {
+      CREATE TABLE t1(a, b);
+      INSERT INTO t1 VALUES(1, 2);
+      CREATE TABLE aux.t2(a, b);
+      INSERT INTO t2 VALUES(1, 2);
+      INSERT INTO t2 VALUES(3, 4);
+    }
+  } {}
+
+  do_test 2.2.$tn.2 { file_page_counts } {1 5 1 7}
+  do_test 2.2.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2}
+  do_test 2.2.$tn.4 { sql1 { PRAGMA wal_checkpoint = RESTART } } {1 5 5}
+  do_test 2.2.$tn.5 { file_page_counts } {2 5 2 7}
+}
+
+do_multiclient_test tn {
+
+  setup_and_attach_aux
+
+  do_test 2.3.$tn.1 {
+    execsql {
+      CREATE TABLE t1(a, b);
+      INSERT INTO t1 VALUES(1, 2);
+      CREATE TABLE aux.t2(a, b);
+      INSERT INTO t2 VALUES(1, 2);
+    }
+  } {}
+
+  do_test 2.3.$tn.2 { file_page_counts } {1 5 1 5}
+  do_test 2.3.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2}
+  do_test 2.3.$tn.4 { sql1 { INSERT INTO t1 VALUES(3, 4) } } {}
+  do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {}
+  do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7}
+
+  do_test 2.3.$tn.7 { sql1 { PRAGMA wal_checkpoint = FULL } } {1 7 5}
+  do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7}
+}
+
+
+
 finish_test