If an attempt to sync the database file as part of a checkpoint fails, do not update the shared "nBackfill" variable. Otherwise, another process could wrap the log and overwrite content before it is synced into the database.

FossilOrigin-Name: b813233d7604a5fd91e1af91d5d812032eec700a

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swhere\san\sSQLITE_BUSY\sin\sthe\scheckpoint\scode\swas\sbeing\streated\sas\san\sIO\serror\s(abandoning,\sinstead\sof\sjust\slimiting,\sthe\scheckpoint).
+C If\san\sattempt\sto\ssync\sthe\sdatabase\sfile\sas\spart\sof\sa\scheckpoint\sfails,\sdo\snot\supdate\sthe\sshared\s"nBackfill"\svariable.\sOtherwise,\sanother\sprocess\scould\swrap\sthe\slog\sand\soverwrite\scontent\sbefore\sit\sis\ssynced\sinto\sthe\sdatabase.
-D 2010-06-04T10:37:06
+D 2010-06-04T11:56:22
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -207,7 +207,7 @@ F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0
 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6
 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
 F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726
-F src/test_vfs.c 6203bc7d88a95a093ceb2c372252317b3154b315
+F src/test_vfs.c d3338794b50d1c777d583fbfa71b41a273684ced
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb
 F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d
@@ -224,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
 F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
-F src/wal.c 453811af926fe12eacea41b4ea02ee1a2d2f004b
+F src/wal.c f34425da57038645102bae4ba1ee2b4a87a06557
 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356
@@ -763,7 +763,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
 F test/wal.test bfec61450b47cdf09f7d2269f9e9967683b8b0fc
 F test/wal2.test 1dcbbe59ab662bebb859bb1ede83143f8a39814e
-F test/wal3.test 535867201f4e5eccc459dbfdbb43e1b9555fd30a
+F test/wal3.test a4b46d20010613e56c8fbb401bc0b370ff838b34
 F test/wal_common.tcl 3e953ae60919281688ea73e4d0aa0e1bc94becd9
 F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432
 F test/walcksum.test 4efa8fb88c32bed8288ea4385a9cc113a5c8f0bf
@@ -817,7 +817,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P df7d59899ceb2743764b0433cb68f4bc33f16344
+P 02c4040ce2b4c970b3dee09f7c9ad5a2a3a9aa49
-R 1a3cb251079fbc88fab0e8991c692f47
+R b9a44acd0cef367d5b42a5c1bc8eb14e
 U dan
-Z 13abdffaf01f3e682d39faa8c6e78666
+Z 578069c872fb1326c91198755fc8278a

manifest.uuid

@@ -1 +1 @@
-02c4040ce2b4c970b3dee09f7c9ad5a2a3a9aa49
+b813233d7604a5fd91e1af91d5d812032eec700a

src/test_vfs.c

@@ -72,7 +72,9 @@ struct Testvfs {
 #define TESTVFS_SHMBARRIER_MASK 0x00000020
 #define TESTVFS_SHMCLOSE_MASK   0x00000040
 
-#define TESTVFS_ALL_MASK        0x0000007F
+#define TESTVFS_OPEN_MASK       0x00000080
+#define TESTVFS_SYNC_MASK       0x00000100
+#define TESTVFS_ALL_MASK        0x000001FF
 
 /*
 ** A shared-memory buffer.
@@ -155,6 +157,86 @@ static sqlite3_io_methods tvfs_io_methods = {
   tvfsShmClose                    /* xShmClose */
 };
 
+static int tvfsResultCode(Testvfs *p, int *pRc){
+  struct errcode {
+    int eCode;
+    const char *zCode;
+  } aCode[] = {
+    { SQLITE_OK,     "SQLITE_OK"     },
+    { SQLITE_ERROR,  "SQLITE_ERROR"  },
+    { SQLITE_IOERR,  "SQLITE_IOERR"  },
+    { SQLITE_LOCKED, "SQLITE_LOCKED" },
+    { SQLITE_BUSY,   "SQLITE_BUSY"   },
+  };
+
+  const char *z;
+  int i;
+
+  z = Tcl_GetStringResult(p->interp);
+  for(i=0; i<ArraySize(aCode); i++){
+    if( 0==strcmp(z, aCode[i].zCode) ){
+      *pRc = aCode[i].eCode;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+static void tvfsExecTcl(
+  Testvfs *p, 
+  const char *zMethod,
+  Tcl_Obj *arg1,
+  Tcl_Obj *arg2,
+  Tcl_Obj *arg3
+){
+  int rc;                         /* Return code from Tcl_EvalObj() */
+  int nArg;                       /* Elements in eval'd list */
+  int nScript;
+  Tcl_Obj ** ap;
+
+  assert( p->pScript );
+
+  if( !p->apScript ){
+    int nByte;
+    int i;
+    if( TCL_OK!=Tcl_ListObjGetElements(p->interp, p->pScript, &nScript, &ap) ){
+      Tcl_BackgroundError(p->interp);
+      Tcl_ResetResult(p->interp);
+      return;
+    }
+    p->nScript = nScript;
+    nByte = (nScript+TESTVFS_MAX_ARGS)*sizeof(Tcl_Obj *);
+    p->apScript = (Tcl_Obj **)ckalloc(nByte);
+    memset(p->apScript, 0, nByte);
+    for(i=0; i<nScript; i++){
+      p->apScript[i] = ap[i];
+    }
+  }
+
+  p->apScript[p->nScript] = Tcl_NewStringObj(zMethod, -1);
+  p->apScript[p->nScript+1] = arg1;
+  p->apScript[p->nScript+2] = arg2;
+  p->apScript[p->nScript+3] = arg3;
+
+  for(nArg=p->nScript; p->apScript[nArg]; nArg++){
+    Tcl_IncrRefCount(p->apScript[nArg]);
+  }
+
+  rc = Tcl_EvalObjv(p->interp, nArg, p->apScript, TCL_EVAL_GLOBAL);
+  if( rc!=TCL_OK ){
+    Tcl_BackgroundError(p->interp);
+    Tcl_ResetResult(p->interp);
+  }
+
+  for(nArg=p->nScript; p->apScript[nArg]; nArg++){
+    Tcl_DecrRefCount(p->apScript[nArg]);
+    p->apScript[nArg] = 0;
+  }
+}
+
+
 /*
 ** Close an tvfs-file.
 */
@@ -208,8 +290,42 @@ static int tvfsTruncate(sqlite3_file *pFile, sqlite_int64 size){
 ** Sync an tvfs-file.
 */
 static int tvfsSync(sqlite3_file *pFile, int flags){
-  TestvfsFile *p = (TestvfsFile *)pFile;
+  int rc = SQLITE_OK;
-  return sqlite3OsSync(p->pReal, flags);
+  TestvfsFile *pFd = (TestvfsFile *)pFile;
+  Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
+
+  if( p->pScript && p->mask&TESTVFS_SYNC_MASK ){
+    char *zFlags;
+
+    switch( flags ){
+      case SQLITE_SYNC_NORMAL:
+        zFlags = "normal";
+        break;
+      case SQLITE_SYNC_FULL:
+        zFlags = "full";
+        break;
+      case SQLITE_SYNC_NORMAL|SQLITE_SYNC_DATAONLY:
+        zFlags = "normal|dataonly";
+        break;
+      case SQLITE_SYNC_FULL|SQLITE_SYNC_DATAONLY:
+        zFlags = "full|dataonly";
+        break;
+      default:
+        assert(0);
+    }
+
+    tvfsExecTcl(p, "xSync", 
+        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId,
+        Tcl_NewStringObj(zFlags, -1)
+    );
+    tvfsResultCode(p, &rc);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = sqlite3OsSync(pFd->pReal, flags);
+  }
+
+  return rc;
 }
 
 /*
@@ -279,14 +395,43 @@ static int tvfsOpen(
   int *pOutFlags
 ){
   int rc;
-  TestvfsFile *p = (TestvfsFile *)pFile;
+  TestvfsFile *pFd = (TestvfsFile *)pFile;
-  p->pShm = 0;
+  Tcl_Obj *pId = 0;
-  p->pShmId = 0;
+  Testvfs *p = (Testvfs *)pVfs->pAppData;
-  p->zFilename = zName;
+
-  p->pVfs = pVfs;
+  pFd->pShm = 0;
-  p->pReal = (sqlite3_file *)&p[1];
+  pFd->pShmId = 0;
-  rc = sqlite3OsOpen(PARENTVFS(pVfs), zName, p->pReal, flags, pOutFlags);
+  pFd->zFilename = zName;
-  if( p->pReal->pMethods ){
+  pFd->pVfs = pVfs;
+  pFd->pReal = (sqlite3_file *)&pFd[1];
+
+  /* Evaluate the Tcl script: 
+  **
+  **   SCRIPT xOpen FILENAME
+  **
+  ** If the script returns an SQLite error code other than SQLITE_OK, an
+  ** error is returned to the caller. If it returns SQLITE_OK, the new
+  ** connection is named "anon". Otherwise, the value returned by the
+  ** script is used as the connection name.
+  */
+  Tcl_ResetResult(p->interp);
+  if( p->pScript && p->mask&TESTVFS_OPEN_MASK ){
+    tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0);
+    if( tvfsResultCode(p, &rc) ){
+      if( rc!=SQLITE_OK ) return rc;
+    }else{
+      pId = Tcl_GetObjResult(p->interp);
+    }
+  }
+  if( !pId ){
+    pId = Tcl_NewStringObj("anon", -1);
+  }
+  Tcl_IncrRefCount(pId);
+  pFd->pShmId = pId;
+  Tcl_ResetResult(p->interp);
+
+  rc = sqlite3OsOpen(PARENTVFS(pVfs), zName, pFd->pReal, flags, pOutFlags);
+  if( pFd->pReal->pMethods ){
     sqlite3_io_methods *pMethods;
     pMethods = (sqlite3_io_methods *)ckalloc(sizeof(sqlite3_io_methods));
     memcpy(pMethods, &tvfs_io_methods, sizeof(sqlite3_io_methods));
@@ -406,84 +551,6 @@ static void tvfsGrowBuffer(TestvfsFile *pFd, int reqSize, int *pNewSize){
   *pNewSize = pBuffer->n;
 }
 
-static void tvfsExecTcl(
-  Testvfs *p, 
-  const char *zMethod,
-  Tcl_Obj *arg1,
-  Tcl_Obj *arg2,
-  Tcl_Obj *arg3
-){
-  int rc;                         /* Return code from Tcl_EvalObj() */
-  int nArg;                       /* Elements in eval'd list */
-  int nScript;
-  Tcl_Obj ** ap;
-
-  assert( p->pScript );
-
-  if( !p->apScript ){
-    int nByte;
-    int i;
-    if( TCL_OK!=Tcl_ListObjGetElements(p->interp, p->pScript, &nScript, &ap) ){
-      Tcl_BackgroundError(p->interp);
-      Tcl_ResetResult(p->interp);
-      return;
-    }
-    p->nScript = nScript;
-    nByte = (nScript+TESTVFS_MAX_ARGS)*sizeof(Tcl_Obj *);
-    p->apScript = (Tcl_Obj **)ckalloc(nByte);
-    memset(p->apScript, 0, nByte);
-    for(i=0; i<nScript; i++){
-      p->apScript[i] = ap[i];
-    }
-  }
-
-  p->apScript[p->nScript] = Tcl_NewStringObj(zMethod, -1);
-  p->apScript[p->nScript+1] = arg1;
-  p->apScript[p->nScript+2] = arg2;
-  p->apScript[p->nScript+3] = arg3;
-
-  for(nArg=p->nScript; p->apScript[nArg]; nArg++){
-    Tcl_IncrRefCount(p->apScript[nArg]);
-  }
-
-  rc = Tcl_EvalObjv(p->interp, nArg, p->apScript, TCL_EVAL_GLOBAL);
-  if( rc!=TCL_OK ){
-    Tcl_BackgroundError(p->interp);
-    Tcl_ResetResult(p->interp);
-  }
-
-  for(nArg=p->nScript; p->apScript[nArg]; nArg++){
-    Tcl_DecrRefCount(p->apScript[nArg]);
-    p->apScript[nArg] = 0;
-  }
-}
-
-static int tvfsResultCode(Testvfs *p, int *pRc){
-  struct errcode {
-    int eCode;
-    const char *zCode;
-  } aCode[] = {
-    { SQLITE_OK,     "SQLITE_OK"     },
-    { SQLITE_ERROR,  "SQLITE_ERROR"  },
-    { SQLITE_IOERR,  "SQLITE_IOERR"  },
-    { SQLITE_LOCKED, "SQLITE_LOCKED" },
-    { SQLITE_BUSY,   "SQLITE_BUSY"   },
-  };
-
-  const char *z;
-  int i;
-
-  z = Tcl_GetStringResult(p->interp);
-  for(i=0; i<ArraySize(aCode); i++){
-    if( 0==strcmp(z, aCode[i].zCode) ){
-      *pRc = aCode[i].eCode;
-      return 1;
-    }
-  }
-
-  return 0;
-}
-
 static int tvfsInjectIoerr(Testvfs *p){
   int ret = 0;
   if( p->ioerr ){
@@ -501,45 +568,30 @@ static int tvfsShmOpen(
 ){
   Testvfs *p;
   int rc = SQLITE_OK;             /* Return code */
-  Tcl_Obj *pId = 0;               /* Id for this connection */
   TestvfsBuffer *pBuffer;         /* Buffer to open connection to */
   TestvfsFile *pFd;               /* The testvfs file structure */
 
   pFd = (TestvfsFile*)pFileDes;
   p = (Testvfs *)pFd->pVfs->pAppData;
-  assert( pFd->pShmId==0 && pFd->pShm==0 );
+  assert( pFd->pShmId && pFd->pShm==0 );
 
   /* Evaluate the Tcl script: 
   **
   **   SCRIPT xShmOpen FILENAME
-  **
-  ** If the script returns an SQLite error code other than SQLITE_OK, an
-  ** error is returned to the caller. If it returns SQLITE_OK, the new
-  ** connection is named "anon". Otherwise, the value returned by the
-  ** script is used as the connection name.
   */
   Tcl_ResetResult(p->interp);
   if( p->pScript && p->mask&TESTVFS_SHMOPEN_MASK ){
     tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0);
     if( tvfsResultCode(p, &rc) ){
       if( rc!=SQLITE_OK ) return rc;
-    }else{
-      pId = Tcl_GetObjResult(p->interp);
     }
   }
-  if( !pId ){
-    pId = Tcl_NewStringObj("anon", -1);
-  }
-  Tcl_IncrRefCount(pId);
 
   assert( rc==SQLITE_OK );
   if( p->mask&TESTVFS_SHMOPEN_MASK && tvfsInjectIoerr(p) ){
-    Tcl_DecrRefCount(pId);
     return SQLITE_IOERR;
   }
 
-  pFd->pShmId = pId;
-
   /* Search for a TestvfsBuffer. Create a new one if required. */
   for(pBuffer=p->pBuffer; pBuffer; pBuffer=pBuffer->pNext){
     if( 0==strcmp(pFd->zFilename, pBuffer->zFile) ) break;
@@ -704,8 +756,6 @@ static int tvfsShmClose(
     ckfree((char *)pBuffer->a);
     ckfree((char *)pBuffer);
   }
-  Tcl_DecrRefCount(pFd->pShmId);
-  pFd->pShmId = 0;
   pFd->pShm = 0;
 
   return rc;
@@ -776,6 +826,8 @@ static int testvfs_obj_cmd(
         { "xShmLock",    TESTVFS_SHMLOCK_MASK },
         { "xShmBarrier", TESTVFS_SHMBARRIER_MASK },
         { "xShmClose",   TESTVFS_SHMCLOSE_MASK },
+        { "xSync",       TESTVFS_SYNC_MASK },
+        { "xOpen",       TESTVFS_OPEN_MASK },
       };
       Tcl_Obj **apElem = 0;
       int nElem = 0;

src/wal.c

@@ -1462,14 +1462,16 @@ static int walCheckpoint(
     }
 
     /* If work was actually accomplished... */
-    if( rc==SQLITE_OK && pInfo->nBackfill<mxSafeFrame ){
+    if( rc==SQLITE_OK ){
-      pInfo->nBackfill = mxSafeFrame;
+      if( mxSafeFrame==pHdr[0].mxFrame ){
-      if( mxSafeFrame==pHdr[0].mxFrame && sync_flags ){
         rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage));
         if( rc==SQLITE_OK && sync_flags ){
           rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
         }
       }
+      if( rc==SQLITE_OK ){
+        pInfo->nBackfill = mxSafeFrame;
+      }
     }
 
     /* Release the reader lock held while backfilling */

test/wal3.test

@@ -172,19 +172,34 @@ foreach code [list {
   } {3 three}
 
   # Try to checkpoint the database using [db]. It should be possible to
-  # checkpoint everything except the frames added by [db3] (checkpointing
+  # checkpoint everything except the table added by [db3] (checkpointing
   # these frames would clobber the snapshot currently being used by [db2]).
   #
+  # After [db2] has committed, a checkpoint can copy the entire log to the
+  # database file. Checkpointing after [db3] has committed is therefore a
+  # no-op, as the entire log has already been backfilled.
+  #
   do_test wal3-2.$tn.4 {
     sql {
       COMMIT;
       PRAGMA wal_checkpoint;
     }
-  } {}
-  do_test wal3-2.$tn.5 {
     file size test.db
   } [expr 3*1024]
-
+  do_test wal3-2.$tn.5 {
+    sql2 {
+      COMMIT;
+      PRAGMA wal_checkpoint;
+    }
+    file size test.db
+  } [expr 4*1024]
+  do_test wal3-2.$tn.6 {
+    sql3 {
+      COMMIT;
+      PRAGMA wal_checkpoint;
+    }
+    file size test.db
+  } [expr 4*1024]
 
   catch { db close }
   catch { code2 { db2 close } }
@@ -192,5 +207,57 @@ foreach code [list {
   catch { close $::code2_chan }
   catch { close $::code3_chan }
 }
+catch {db close}
+
+#-------------------------------------------------------------------------
+# Test that that for the simple test:
+#
+#   CREATE TABLE x(y);
+#   INSERT INTO x VALUES('z');
+#   PRAGMA wal_checkpoint;
+#
+# in WAL mode the xSync method is invoked as expected for each of
+# synchronous=off, synchronous=normal and synchronous=full.
+#
+foreach {tn syncmode synccount} {
+  1 off     
+    {}
+  2 normal  
+    {test.db-wal normal test.db normal}
+  3 full    
+    {test.db-wal normal test.db-wal normal test.db-wal normal test.db normal}
+} {
+
+  proc sync_counter {args} { 
+    foreach {method filename id flags} $args break
+    lappend ::syncs [file tail $filename] $flags
+  }
+  do_test wal3-3.$tn {
+    file delete -force test.db test.db-wal test.db-journal
+  
+    testvfs T
+    T filter {} 
+    T script sync_counter
+    sqlite3 db test.db -vfs T
+  
+    execsql "PRAGMA synchronous = $syncmode"
+    execsql { PRAGMA journal_mode = WAL }
+
+    set ::syncs [list]
+    T filter xSync
+    execsql {
+      CREATE TABLE x(y);
+      INSERT INTO x VALUES('z');
+      PRAGMA wal_checkpoint;
+    }
+    T filter {}
+    set ::syncs
+  } $synccount
+
+  db close
+  T delete
+}
+
 finish_test
 
+