Fix for 'truncate file' operations on in-memory databases. (CVS 6131)

FossilOrigin-Name: 83d1eafbde556f56969c6f285b6767d2c658dbfc

manifest

@@ -1,5 +1,5 @@
-C Add\sa\scomment\sto\sthe\sopenSubjournal()\sfunction\sin\spager.c.\s(CVS\s6130)
+C Fix\sfor\s'truncate\sfile'\soperations\son\sin-memory\sdatabases.\s(CVS\s6131)
-D 2009-01-07T10:52:56
+D 2009-01-07T15:18:21
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 05461a9b5803d5ad10c79f989801e9fd2cc3e592
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -142,12 +142,12 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60
 F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5
 F src/os_unix.c e6eacc7ec735ded605fefcbaf250058baa8feb12
 F src/os_win.c 496e3ceb499aedc63622a89ef76f7af2dd902709
-F src/pager.c 1af21dadcfea3050b34ec54a5a173da368e13e10
+F src/pager.c a799290e5cf9495f9991413ce84847011767b4d9
-F src/pager.h 0793c5e4faed6c278037eb22b2434b318687d615
+F src/pager.h 9870acb2d653848d90d765d7cbf163496d6c8111
 F src/parse.y 4d0e33a702dc3ea7b69d8ae1914b3fbd32e46057
 F src/pcache.c 16dc8da6e6ba6250f8dfd9ee46036db1cbceedc6
 F src/pcache.h f20c3e82dd6da622c3fe296170cb1801f9a2d75a
-F src/pcache1.c 533b18aa2456b0f135e376289443e0a342e0c456
+F src/pcache1.c c0aa84ff69ea759fa944dbee9167a2463ab7c322
 F src/pragma.c d6dfc47979623cf593d0db926daff49d214a8eb3
 F src/prepare.c 51d11eb1eece7548038e7e33c33ee95dc7f82e2d
 F src/printf.c 9866a9a9c4a90f6d4147407f373df3fd5d5f9b6f
@@ -164,7 +164,7 @@ F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
 F src/table.c 23db1e5f27c03160987c122a078b4bb51ef0b2f8
 F src/tclsqlite.c 4415e1033bd3e92b05a6a9cde911ee4de3b82df9
 F src/test1.c b193b8b80617bdb8297b25a87d00ee8d5a125d0d
-F src/test2.c 15dbd8b99a05017eaafee9840c4c9dd63d26697a
+F src/test2.c 724095fc69cb54b018aaa4463a56836c0b53a17a
 F src/test3.c 88a246b56b824275300e6c899634fbac1dc94b14
 F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c
 F src/test5.c 162a1cea2105a2c460a3f39fa6919617b562a288
@@ -449,7 +449,7 @@ F test/mallocI.test 6e24fe6444bd2999ccc81f984977b44c0d6e5591
 F test/mallocJ.test 44dfbbaca731cb933818ad300b4566265d652609
 F test/malloc_common.tcl 984baeb6c6b185e798827d1187d426acc2bc4962
 F test/manydb.test b3d3bc4c25657e7f68d157f031eb4db7b3df0d3c
-F test/memdb.test a67bda4ff90a38f2b19f6c7f95aa7289e051d893
+F test/memdb.test a67c85a29d3187ac81d3628fcf9417d8a1be9ecb
 F test/memleak.test d2d2a1ff7105d32dc3fdf691458cf6cba58c7217
 F test/memsubsys1.test bdc24a38d198679d777ca4efcc089fd3948567a6
 F test/memsubsys2.test 72a731225997ad5e8df89fdbeae9224616b6aecc
@@ -470,7 +470,7 @@ F test/nan.test c627d79b3d36ea892563fd67584b3e8a18f0618a
 F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82
 F test/null.test a8b09b8ed87852742343b33441a9240022108993
 F test/openv2.test f5dd6b23e4dce828eb211649b600763c42a668df
-F test/pager.test ef306077e282963664be71ac31832f4476e557ca
+F test/pager.test 7e1788cce372ca0302a787d6b44821e269d10790
 F test/pager2.test d4b7f6b70ff018b9995e622a32526b275f515042
 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4
 F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a
@@ -692,7 +692,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P 3a4bb83235e9a79297e7d5d47ac7674c9df960bf
+P 04387ae10ab3be24c93497f4af6f48d6832f37eb
-R 69328c811bc9ff20117c3aeb0aaeffc8
+R cd03e1b03d4149ac01b912dddfeb4003
 U danielk1977
-Z 001298448438542bba82708599474b69
+Z 242d556585dea66787f6cacf6e86748b

manifest.uuid

@@ -1 +1 @@
-04387ae10ab3be24c93497f4af6f48d6832f37eb
+83d1eafbde556f56969c6f285b6767d2c658dbfc

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.537 2009/01/07 10:52:56 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.538 2009/01/07 15:18:21 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -1086,6 +1086,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){
   pPager->setMaster = 0;
   pPager->needSync = 0;
   /* lruListSetFirstSynced(pPager); */
+  sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
   if( !MEMDB ){
     pPager->dbSizeValid = 0;
   }
@@ -1462,10 +1463,11 @@ delmaster_out:
 
 
 /*
-** Truncate the main file of the given pager to the number of pages
+** If the main database file is open and an exclusive lock is held, 
-** indicated. Also truncate the cached representation of the file.
+** truncate the main file of the given pager to the specified number 
+** of pages.
 **
-** Might might be the case that the file on disk is smaller than nPage.
+** It might might be the case that the file on disk is smaller than nPage.
 ** This can happen, for example, if we are in the middle of a transaction
 ** which has extended the file size and the new pages are still all held
 ** in cache, then an INSERT or UPDATE does a statement rollback.  Some
@@ -1491,10 +1493,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
       }
     }
   }
-  if( rc==SQLITE_OK ){
-    pPager->dbSize = nPage;
-    sqlite3PcacheTruncate(pPager->pPCache, nPage);
-  }
   return rc;
 }
 
@@ -1668,6 +1666,7 @@ static int pager_playback(Pager *pPager, int isHot){
       if( rc!=SQLITE_OK ){
         goto end_playback;
       }
+      pPager->dbSize = mxPg;
     }
 
     /* Copy original pages out of the journal and back into the database file.
@@ -2349,41 +2348,12 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){
   return rc;
 }
 
-/*
-** Truncate the file to the number of pages specified. 
-**
-** Unless an IO error occurs, this function is guaranteed to modify the 
-** database file itself. If an exclusive lock is not held when this function
-** is called, one is obtained before truncating the file.
-*/
-int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
-  int rc = SQLITE_OK;
-  assert( pPager->state>=PAGER_SHARED );
-
-  sqlite3PagerPagecount(pPager, 0);
-  if( pPager->errCode ){
-    rc = pPager->errCode;
-  }else if( nPage<pPager->dbFileSize ){
-    rc = syncJournal(pPager);
-    if( rc==SQLITE_OK ){
-      /* Get an exclusive lock on the database before truncating. */
-      rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pager_truncate(pPager, nPage);
-    }
-  }
-
-  return rc;
-}
-
 #ifndef SQLITE_OMIT_AUTOVACUUM
 /*
-** Truncate the in-memory database file image to nPage pages. Unlike
+** Truncate the in-memory database file image to nPage pages. This 
-** sqlite3PagerTruncate(), this function does not actually modify the
+** function does not actually modify the database file on disk. It 
-** database file on disk. It just sets the internal state of the pager
+** just sets the internal state of the pager object so that the 
-** object so that the truncation will be done when the current 
+** truncation will be done when the current transaction is committed.
-** transaction is committed.
 */
 void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){
   assert( pPager->dbSizeValid );
@@ -2606,7 +2576,7 @@ static int pager_write_pagelist(PgHdr *pList){
     }
 
     /* If there are dirty pages in the page cache with page numbers greater
-    ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to
+    ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
     ** make the file smaller (presumably by auto-vacuum code). Do not write
     ** any such pages to the file.
     */
@@ -3963,13 +3933,6 @@ int sqlite3PagerCommitPhaseOne(
     }
     if( rc!=SQLITE_OK ) goto sync_exit;
 
-#ifndef SQLITE_OMIT_AUTOVACUUM
-    if( pPager->dbSize<pPager->dbFileSize ){
-      rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
-      if( rc!=SQLITE_OK ) goto sync_exit;
-    }
-#endif
-
     /* Write all dirty pages to the database file */
     pPg = sqlite3PcacheDirtyList(pPager->pPCache);
     rc = pager_write_pagelist(pPg);
@@ -3986,6 +3949,12 @@ int sqlite3PagerCommitPhaseOne(
     }
     sqlite3PcacheCleanAll(pPager->pPCache);
 
+    if( pPager->dbSize<pPager->dbFileSize ){
+      assert( pPager->state>=PAGER_EXCLUSIVE );
+      rc = pager_truncate(pPager, pPager->dbSize);
+      if( rc!=SQLITE_OK ) goto sync_exit;
+    }
+
     /* Sync the database file. */
     if( !pPager->noSync && !noSync ){
       rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
@@ -3993,8 +3962,6 @@ int sqlite3PagerCommitPhaseOne(
     IOTRACE(("DBSYNC %p\n", pPager))
 
     pPager->state = PAGER_SYNCED;
-  }else if( MEMDB && pPager->dbSize<pPager->dbFileSize ){
-    rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
   }
 
 sync_exit:

src/pager.h

@@ -13,7 +13,7 @@
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.92 2009/01/06 14:19:37 drh Exp $
+** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $
 */
 
 #ifndef _PAGER_H_
@@ -88,7 +88,6 @@ int sqlite3PagerRef(DbPage*);
 int sqlite3PagerUnref(DbPage*);
 int sqlite3PagerWrite(DbPage*);
 int sqlite3PagerPagecount(Pager*, int*);
-int sqlite3PagerTruncate(Pager*,Pgno);
 int sqlite3PagerBegin(DbPage*, int exFlag);
 int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int);
 int sqlite3PagerCommitPhaseTwo(Pager*);

src/pcache1.c

@@ -16,7 +16,7 @@
 ** If the default page cache implementation is overriden, then neither of
 ** these two features are available.
 **
-** @(#) $Id: pcache1.c,v 1.6 2008/12/10 18:03:46 drh Exp $
+** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $
 */
 
 #include "sqliteInt.h"
@@ -47,6 +47,8 @@ struct PCache1 {
   unsigned int nPage;                 /* Total number of pages in apHash */
   unsigned int nHash;                 /* Number of slots in apHash[] */
   PgHdr1 **apHash;                    /* Hash table for fast lookup by key */
+
+  unsigned int iMaxKey;               /* Largest key seen since xTruncate() */
 };
 
 /*
@@ -557,6 +559,9 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
   }
 
 fetch_out:
+  if( pPage && iKey>pCache->iMaxKey ){
+    pCache->iMaxKey = iKey;
+  }
   if( createFlag==1 ) sqlite3EndBenignMalloc();
   pcache1LeaveMutex();
   return (pPage ? PGHDR1_TO_PAGE(pPage) : 0);
@@ -632,6 +637,10 @@ static void pcache1Rekey(
   pPage->pNext = pCache->apHash[h];
   pCache->apHash[h] = pPage;
 
+  if( iNew>pCache->iMaxKey ){
+    pCache->iMaxKey = iNew;
+  }
+
   pcache1LeaveMutex();
 }
 
@@ -645,7 +654,10 @@ static void pcache1Rekey(
 static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
   PCache1 *pCache = (PCache1 *)p;
   pcache1EnterMutex();
+  if( iLimit<=pCache->iMaxKey ){
     pcache1TruncateUnsafe(pCache, iLimit);
+    pCache->iMaxKey = iLimit-1;
+  }
   pcache1LeaveMutex();
 }
 

src/test2.c

@@ -13,7 +13,7 @@
 ** is not included in the SQLite library.  It is used for automated
 ** testing of the SQLite library.
 **
-** $Id: test2.c,v 1.64 2009/01/02 18:10:42 drh Exp $
+** $Id: test2.c,v 1.65 2009/01/07 15:18:21 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -393,7 +393,6 @@ static int pager_truncate(
   const char **argv      /* Text of each argument */
 ){
   Pager *pPager;
-  int rc;
   int pgno;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
@@ -402,11 +401,7 @@ static int pager_truncate(
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   if( Tcl_GetInt(interp, argv[2], &pgno) ) return TCL_ERROR;
-  rc = sqlite3PagerTruncate(pPager, pgno);
+  sqlite3PagerTruncateImage(pPager, pgno);
-  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, errorName(rc), 0);
-    return TCL_ERROR;
-  }
   return TCL_OK;
 }
 

test/memdb.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is in-memory database backend.
 #
-# $Id: memdb.test,v 1.15 2006/01/30 22:48:44 drh Exp $
+# $Id: memdb.test,v 1.16 2009/01/07 15:18:21 danielk1977 Exp $
 
 
 set testdir [file dirname $argv0]
@@ -411,6 +411,25 @@ do_test memdb-8.2 {
   }
 } 0
 
+# Test that auto-vacuum works with in-memory databases.
+# 
+do_test memdb-9.1 {
+  db close
+  sqlite3 db test.db
+  db cache size 0
+  execsql {
+    PRAGMA auto_vacuum = full;
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES(randstr(1000,1000));
+    INSERT INTO t1 VALUES(randstr(1000,1000));
+    INSERT INTO t1 VALUES(randstr(1000,1000));
+  }
+  set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
+  execsql { DELETE FROM t1 }
+  set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
+  expr {$memused2 + 2048 < $memused}
+} {1}
+
 
 } ;# ifcapable memorydb
 

test/pager.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is page cache subsystem.
 #
-# $Id: pager.test,v 1.33 2009/01/05 17:15:00 danielk1977 Exp $
+# $Id: pager.test,v 1.34 2009/01/07 15:18:21 danielk1977 Exp $
 
 
 set testdir [file dirname $argv0]
@@ -413,7 +413,7 @@ do_test pager-4.6.1 {
 ifcapable memorydb {
   do_test pager-4.6.2 {
     set ::p2 [pager_open :memory: 10]
-    pager_truncate $::p2 5
+    pager_truncate $::p2 0
   } {}
   do_test pager-4.6.3 {
     set page1 [page_get $::p2 1]
@@ -424,7 +424,7 @@ ifcapable memorydb {
       page_unref $p
     }
     page_unref $page1
-    # pager_truncate $::p2 3
+    pager_truncate $::p2 3
   } {}
   do_test pager-4.6.4 {
     pager_close $::p2