Merge the fix for [a1fa75cbdd02] from the experimental branch. Also

fix the persistent-wal mode feature of truncating the WAL on close so that
it always truncates the WAL to zero bytes.

FossilOrigin-Name: 09ccc4a1be7ba81890f10aac6623dd90dab4f990

manifest

@@ -1,5 +1,5 @@
-C Improved\slogging\sof\smaster-journal\sname\sconflicts.
+C Merge\sthe\sfix\sfor\s[a1fa75cbdd02]\sfrom\sthe\sexperimental\sbranch.\s\sAlso\nfix\sthe\spersistent-wal\smode\sfeature\sof\struncating\sthe\sWAL\son\sclose\sso\sthat\nit\salways\struncates\sthe\sWAL\sto\szero\sbytes.
-D 2011-12-16T15:11:39.468
+D 2011-12-16T15:38:52.854
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -250,7 +250,7 @@ F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56
 F src/vdbesort.c 468d43c057063e54da4f1988b38b4f46d60e7790
 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843
 F src/vtab.c e9318d88feac85be8e27ee783ac8f5397933fc8a
-F src/wal.c 7e6e7fe68ee649505dca38c8ab83eda0d0d96ae5
+F src/wal.c 8575f2bdaed48e7ffbea8608b614bb7abd382a54
 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c af623942514571895818b9b7ae11db95ae3b3d88
@@ -914,11 +914,12 @@ F test/walbig.test 0ab8a430ef420a3114f7092e0f30fc9585ffa155
 F test/walcksum.test f5447800a157c9e2234fbb8e80243f0813941bde
 F test/walcrash.test 4fcb661faf71db91214156d52d43ee327f52bde1
 F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142
+F test/walcrash3.test 26d251a6dfaa1daae983268ccc6da0521d61e2e7
 F test/walfault.test efb0d5724893133e71b8d9d90abdb781845a6bb0
 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
 F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
-F test/walpersist.test 710b1b6cf6f8333e984f437724d1fa9e0511c5aa
+F test/walpersist.test 4d308b7f16470e85cb389636e59c455fe6b3a61a
 F test/walro.test e6bb27762c9f22601cbb8bff6e0acfd124e74b63
 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761
 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a
@@ -983,7 +984,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P cf3bccc2e944cd2dd3efb8554682994a06115f16
+P b1005ef46cc2b46dd8e448ae1a9a9508bd5666ab 6492af76ea6585a1b377d69751af930c0ccfe688
-R 9cdd098939326e460c36a0ff01e82e7f
+R 5e90bbe7156facb03e2de52b2217bc8c
 U drh
-Z f579d2bc7a13e23fcc9e8ceaf02a267f
+Z 290df3a9d2eb83a09d72442af4b2f315

manifest.uuid

@@ -1 +1 @@
-b1005ef46cc2b46dd8e448ae1a9a9508bd5666ab
+09ccc4a1be7ba81890f10aac6623dd90dab4f990

src/wal.c

@@ -421,6 +421,7 @@ struct Wal {
   u8 writeLock;              /* True if in a write transaction */
   u8 ckptLock;               /* True if holding a checkpoint lock */
   u8 readOnly;               /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */
+  u8 truncateOnCommit;       /* True to truncate WAL file on commit */
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
@@ -1782,22 +1783,20 @@ static int walCheckpoint(
 }
 
 /*
-** Attempt to limit the WAL size to the size limit defined by
+** If the WAL file is currently larger than nMax bytes in size, truncate
-** PRAGMA journal_size_limit.
+** it to exactly nMax bytes. If an error occurs while doing so, ignore it.
 */
-static void walLimitSize(Wal *pWal){
+static void walLimitSize(Wal *pWal, i64 nMax){
-  if( pWal->mxWalSize>=0 ){
+  i64 sz;
-    i64 sz;
+  int rx;
-    int rx;
+  sqlite3BeginBenignMalloc();
-    sqlite3BeginBenignMalloc();
+  rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
-    rx = sqlite3OsFileSize(pWal->pWalFd, &sz);
+  if( rx==SQLITE_OK && (sz > nMax ) ){
-    if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){
+    rx = sqlite3OsTruncate(pWal->pWalFd, nMax);
-      rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize);
+  }
-    }
+  sqlite3EndBenignMalloc();
-    sqlite3EndBenignMalloc();
+  if( rx ){
-    if( rx ){
+    sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
-      sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName);
-    }
   }
 }
 
@@ -1824,18 +1823,29 @@ int sqlite3WalClose(
     */
     rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
     if( rc==SQLITE_OK ){
-      int bPersistWal = -1;
       if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
       rc = sqlite3WalCheckpoint(
           pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
       );
-      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal);
+      if( rc==SQLITE_OK ){
-      if( rc==SQLITE_OK && bPersistWal!=1 ){
+        int bPersist = -1;
-        isDelete = 1;
+        sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist);
-      }else{
+        if( bPersist!=1 ){
-        walLimitSize(pWal);
+          /* Try to delete the WAL file if the checkpoint completed and
+          ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
+          ** mode (!bPersist) */
+          isDelete = 1;
+        }else if( pWal->mxWalSize>=0 ){
+          /* Try to truncate the WAL file to zero bytes if the checkpoint
+          ** completed and fsynced (rc==SQLITE_OK) and we are in persistent
+          ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a
+          ** non-negative value (pWal->mxWalSize>=0).  Note that we truncate
+          ** to zero bytes as truncating to the journal_size_limit might
+          ** leave a corrupt WAL file on disk. */
+          walLimitSize(pWal, 0);
+        }
       }
     }
 
@@ -2444,6 +2454,7 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){
   if( pWal->writeLock ){
     walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
     pWal->writeLock = 0;
+    pWal->truncateOnCommit = 0;
   }
   return SQLITE_OK;
 }
@@ -2578,7 +2589,6 @@ static int walRestartLog(Wal *pWal){
         int i;                    /* Loop counter */
         u32 *aSalt = pWal->hdr.aSalt;       /* Big-endian salt values */
 
-        walLimitSize(pWal);
         pWal->nCkpt++;
         pWal->hdr.mxFrame = 0;
         sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0]));
@@ -2666,6 +2676,7 @@ int sqlite3WalFrames(
     pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
     pWal->hdr.aFrameCksum[0] = aCksum[0];
     pWal->hdr.aFrameCksum[1] = aCksum[1];
+    pWal->truncateOnCommit = 1;
 
     rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
     WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok"));
@@ -2739,6 +2750,15 @@ int sqlite3WalFrames(
     rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
   }
 
+  if( isCommit && pWal->truncateOnCommit && pWal->mxWalSize>=0 ){
+    i64 sz = pWal->mxWalSize;
+    if( walFrameOffset(iFrame+nLast+1, szPage)>pWal->mxWalSize ){
+      sz = walFrameOffset(iFrame+nLast+1, szPage);
+    }
+    walLimitSize(pWal, sz);
+    pWal->truncateOnCommit = 0;
+  }
+
   /* Append data to the wal-index. It is not necessary to lock the 
   ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index
   ** guarantees that there are no other writers, and no data that may

test/walcrash3.test

@@ -0,0 +1,75 @@
+# 2011 December 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This test simulates an application crash immediately following a
+# system call to truncate a file. Specifically, the system call that
+# truncates the WAL file if "PRAGMA journal_size_limit" is configured.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !wal {finish_test ; return }
+set testprefix walcrash3
+
+db close
+testvfs tvfs
+tvfs filter {xTruncate xWrite}
+tvfs script tvfs_callback
+proc tvfs_callback {args} {}
+
+sqlite3 db test.db -vfs tvfs
+do_execsql_test 1.1 {
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = WAL;
+  PRAGMA wal_autocheckpoint = 128;
+  PRAGMA journal_size_limit = 16384;
+
+  CREATE TABLE t1(a BLOB, b BLOB, UNIQUE(a, b));
+  INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
+} {wal 128 16384}
+
+proc tvfs_callback {method file arglist} {
+  if {$::state==1} {
+    foreach f [glob -nocomplain xx_test.*] { forcedelete $f }
+    foreach f [glob -nocomplain test.*]    { forcecopy $f "xx_$f" }
+    set ::state 2
+  }
+  if {$::state==0 && $method=="xTruncate" && [file tail $file]=="test.db-wal"} {
+    set ::state 1
+  }
+}
+
+for {set i 2} {$i<1000} {incr i} {
+
+  # If the WAL file is truncated within the following, within the following
+  # xWrite call the [tvfs_callback] makes a copy of the database and WAL 
+  # files set sets $::state to 2. So that the copied files are in the same
+  # state as the real database and WAL files would be if an application crash 
+  # occurred immediately following the xTruncate().
+  # 
+  set ::state 0
+  do_execsql_test 1.$i.1 {
+    INSERT INTO t1 VALUES(randomblob(10), randomblob(1000));
+  }
+
+  # If a copy was made, open it and run the integrity-check.
+  #
+  if {$::state==2} {
+    sqlite3 db2 xx_test.db
+    do_test 1.$i.2 { execsql { PRAGMA integrity_check  } db2 } "ok"
+    do_test 1.$i.3 { execsql { SELECT count(*) FROM t1 } db2 } [expr $i-1]
+    db2 close
+  }
+}
+
+finish_test
+

test/walpersist.test

@@ -68,7 +68,10 @@ do_test walpersist-1.11 {
 } {1 1 1}
 
 # Make sure the journal_size_limit works to limit the size of the
-# persisted wal file.
+# persisted wal file.  In persistent-wal mode, any non-negative
+# journal_size_limit causes the WAL file to be truncated to zero bytes
+# when closing.
+#
 forcedelete test.db test.db-shm test.db-wal
 do_test walpersist-2.1 {
   sqlite3 db test.db
@@ -85,7 +88,7 @@ do_test walpersist-2.1 {
 do_test walpersist-2.2 {
   file_control_persist_wal db 1
   db close
-  file size test.db-wal
+  concat [file exists test.db-wal] [file size test.db-wal]
-} {12000}
+} {1 0}
 
 finish_test