Create a version of the log checksummer that works on big-endian platforms. Remove the 512KB size limit on the log-summary.

FossilOrigin-Name: 5d6d4423d1def39bd2424703120aa985085c3f8e

manifest

@@ -1,5 +1,5 @@
-C Further\stests\sand\schanges\srelated\sto\sswitching\sbetween\sWAL\sand\srollback\smodes.
+C Create\sa\sversion\sof\sthe\slog\schecksummer\sthat\sworks\son\sbig-endian\splatforms.\sRemove\sthe\s512KB\ssize\slimit\son\sthe\slog-summary.
-D 2010-04-22T06:27:05
+D 2010-04-22T19:14:14
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
 F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581
-F src/log.c edafa806982a281205e71c439e99766e924de938
+F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6
 F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb
 F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e
 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a
@@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30
 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f
 F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1
 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053
-F src/pager.c 06fb9b8b7c5283c62f5258e098d6d1649dde7635
+F src/pager.c d83d2ea6d2025554e079f891854b133924305e19
 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444
 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
 F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf
@@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
-F test/wal.test fd0e5914599b205172dae33c765368a75feb2462
+F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49
 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1
 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47
 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df
@@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 9f4f933f2c6596064fcfc7fb5add87e8c5b57448
+P 1236318477787a612f02cc98caf2931bd2e99d94
-R cdccc9135123b3edc2fcb17c83fbd857
+R 2eba2ad72744173c34347863efb93bd2
 U dan
-Z 891f9ffc6892beb06734f78c0f8e5711
+Z b18c4f1fb26d4b385c95cfa4edf72422

manifest.uuid

@@ -1 +1 @@
-1236318477787a612f02cc98caf2931bd2e99d94
+5d6d4423d1def39bd2424703120aa985085c3f8e

src/log.c

@@ -83,6 +83,15 @@ struct LogSummaryHdr {
   LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE)              \
 )
 
+/*
+** If using mmap() to access a shared (or otherwise) log-summary file, then
+** the mapping size is incremented in units of the following size.
+**
+** A 64 KB log-summary mapping corresponds to a log file containing over
+** 13000 frames, so the mapping size does not need to be increased often.
+*/
+#define LOGSUMMARY_MMAP_INCREMENT (64*1024)
+
 /*
 ** There is one instance of this structure for each log-summary object
 ** that this process has a connection to. They are stored in a linked
@@ -99,6 +108,7 @@ struct LogSummary {
   char *zPath;                    /* Path to associated WAL file */
   LogLock *pLock;                 /* Linked list of locks on this object */
   LogSummary *pNext;              /* Next in global list */
+
   int nData;                      /* Size of aData allocation/mapping */
   u32 *aData;                     /* File body */
 };
@@ -288,10 +298,22 @@ static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){
   assert( LOG_CKSM_BYTES==2*sizeof(u32) );
   assert( (nByte&0x00000003)==0 );
 
+  if( SQLITE_LITTLEENDIAN ){
+#ifdef SQLITE_DEBUG
+    u8 *a = (u8 *)a32;
+    assert( *a32==(a[0] + (a[1]<<8) + (a[2]<<16) + (a[3]<<24)) );
+#endif
     do {
-    sum1 += (*a32++);
+      sum1 += *a32;
       sum2 += sum1;
-  } while( a32<aEnd );
+    } while( ++a32<aEnd );
+  }else{
+    do {
+      u8 *a = (u8*)a32;
+      sum1 += a[0] + (a[1]<<8) + (a[2]<<16) + (a[3]<<24);
+      sum2 += sum1;
+    } while( ++a32<aEnd );
+  }
 
   aCksum[0] = sum1 + (sum1>>24);
   aCksum[1] = sum2 + (sum2>>24);
@@ -333,42 +355,6 @@ static void logNormalizePath(char *zPath){
   z[j] = 0;
 }
 
-/*
-** Memory map the first nByte bytes of the summary file opened with 
-** pSummary->fd at pSummary->aData. If the summary file is smaller than
-** nByte bytes in size when this function is called, ftruncate() is
-** used to expand it before it is mapped.
-**
-** It is assumed that an exclusive lock is held on the summary file
-** by the caller (to protect the ftruncate()).
-*/
-static int logSummaryMap(LogSummary *pSummary, int nByte){
-  struct stat sStat;
-  int rc;
-  int fd = pSummary->fd;
-  void *pMap;
-
-  assert( pSummary->aData==0 );
-
-  /* If the file is less than nByte bytes in size, cause it to grow. */
-  rc = fstat(fd, &sStat);
-  if( rc!=0 ) return SQLITE_IOERR;
-  if( sStat.st_size<nByte ){
-    rc = ftruncate(fd, nByte);
-    if( rc!=0 ) return SQLITE_IOERR;
-  }
-
-  /* Map the file. */
-  pMap = mmap(0, nByte, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-  if( pMap==MAP_FAILED ){
-    return SQLITE_IOERR;
-  }
-  pSummary->aData = (u32 *)pMap;
-  pSummary->nData = nByte;
-
-  return SQLITE_OK;
-}
-
 /*
 ** Unmap the log-summary mapping and close the file-descriptor. If
 ** the isTruncate argument is non-zero, truncate the log-summary file
@@ -523,6 +509,44 @@ static void logMergesort8(
 }
 
 
+/*
+** Memory map the first nByte bytes of the summary file opened with 
+** pSummary->fd at pSummary->aData. If the summary file is smaller than
+** nByte bytes in size when this function is called, ftruncate() is
+** used to expand it before it is mapped.
+**
+** It is assumed that an exclusive lock is held on the summary file
+** by the caller (to protect the ftruncate()).
+*/
+static int logSummaryMap(LogSummary *pSummary, int nByte){
+  struct stat sStat;
+  int rc;
+  int fd = pSummary->fd;
+  void *pMap;
+
+  assert( pSummary->aData==0 );
+
+  /* If the file is less than nByte bytes in size, cause it to grow. */
+  rc = fstat(fd, &sStat);
+  if( rc!=0 ) return SQLITE_IOERR;
+  if( sStat.st_size<nByte ){
+    rc = ftruncate(fd, nByte);
+    if( rc!=0 ) return SQLITE_IOERR;
+  }else{
+    nByte = sStat.st_size;
+  }
+
+  /* Map the file. */
+  pMap = mmap(0, nByte, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+  if( pMap==MAP_FAILED ){
+    return SQLITE_IOERR;
+  }
+  pSummary->aData = (u32 *)pMap;
+  pSummary->nData = nByte/4;
+
+  return SQLITE_OK;
+}
+
 /*
 ** Return the index in the LogSummary.aData array that corresponds to 
 ** frame iFrame. The log-summary file consists of a header, followed by
@@ -543,6 +567,16 @@ static int logSummaryEntry(u32 iFrame){
 static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
   u32 iSlot = logSummaryEntry(iFrame);
 
+  if( (iSlot+128)>=pSummary->nData ){
+    int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;
+
+    sqlite3_mutex_enter(pSummary->mutex);
+    munmap(pSummary->aData, pSummary->nData*4);
+    pSummary->aData = 0;
+    logSummaryMap(pSummary, nByte);
+    sqlite3_mutex_leave(pSummary->mutex);
+  }
+
   /* Set the log-summary entry itself */
   pSummary->aData[iSlot] = iPage;
 
@@ -606,7 +640,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){
     ** SQLITE_MAX_PAGE_SIZE, conclude that the log file contains no valid data.
     */
     nPgsz = sqlite3Get4byte(&aBuf[0]);
-    if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){
+    if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE || nPgsz<512 ){
       goto finished;
     }
     aCksum[0] = sqlite3Get4byte(&aBuf[4]);
@@ -885,7 +919,7 @@ static int logSummaryInit(
   assert( sqlite3_mutex_held(pSummary->mutex) );
   rc = logLockMutex(pSummary, LOG_WRLOCKW);
   if( rc!=SQLITE_OK ) return rc;
-  rc = logSummaryMap(pSummary, 512*1024);
+  rc = logSummaryMap(pSummary, LOGSUMMARY_MMAP_INCREMENT);
   if( rc!=SQLITE_OK ) goto out;
 
   /* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this

src/pager.c

@@ -3755,8 +3755,8 @@ static int pagerHasWAL(Pager *pPager, int *pExists){
 }
 
 static int pagerOpenSnapshot(Pager *pPager){
-  int rc;
+  int rc;                         /* Return code */
-  int changed;
+  int changed = 0;                /* True if cache must be reset */
 
   assert( pagerUseLog(pPager) );
 

test/wal.test

@@ -676,7 +676,6 @@ do_test wal-12.3 {
   execsql { INSERT INTO t2 VALUES('B', 1) }
   list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044]
 } {3 2}
-
 do_test wal-12.4 {
   file copy -force test.db test2.db
   file copy -force test.db-wal test2.db-wal
@@ -684,7 +683,6 @@ do_test wal-12.4 {
   execsql { SELECT * FROM t2 } db2
 } {B 1}
 db2 close
-
 do_test wal-12.5 {
   execsql {
     PRAGMA checkpoint;
@@ -696,14 +694,47 @@ do_test wal-12.5 {
     SELECT * FROM t2;
   }
 } {B 2}
-
+do_test wal-12.6 {
-do_test wal-12.4 {
   file copy -force test.db test2.db
   file copy -force test.db-wal test2.db-wal
   sqlite3_wal db2 test2.db
   execsql { SELECT * FROM t2 } db2
 } {B 2}
 db2 close
+db close
+
+#-------------------------------------------------------------------------
+# Test large log summaries.
+#
+do_test wal-13.1 {
+  list [file exists test.db] [file exists test.db-wal]
+} {1 0}
+do_test wal-13.2 {
+  set fd [open test.db-wal w]
+  seek $fd [expr 200*1024*1024]
+  puts $fd ""
+  close $fd
+  sqlite3 db test.db
+  execsql { SELECT * FROM t2 }
+} {B 2}
+do_test wal-13.3 {
+  db close
+  file exists test.db-wal
+} {0}
+do_test wal-13.4 {
+  sqlite3 db test.db
+  execsql { SELECT count(*) FROM t2 }
+} {1}
+do_test wal-13.5 {
+  for {set i 0} {$i < 15} {incr i} {
+    execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
+  }
+  execsql { SELECT count(*) FROM t2 }
+} [expr int(pow(2, 15))]
+do_test wal-13.6 {
+  file size test.db-wal-summary
+} [expr 192*1024]
+
 
 finish_test