From f90b7260b31638ca2cf8f5ec4b33f8c4b621baeb Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Wed, 7 Jan 2009 15:18:20 +0000 Subject: [PATCH] Fix for 'truncate file' operations on in-memory databases. (CVS 6131) FossilOrigin-Name: 83d1eafbde556f56969c6f285b6767d2c658dbfc --- manifest | 22 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 69 +++++++++++++------------------------------------ src/pager.h | 3 +-- src/pcache1.c | 16 ++++++++++-- src/test2.c | 9 ++----- test/memdb.test | 21 ++++++++++++++- test/pager.test | 6 ++--- 8 files changed, 70 insertions(+), 78 deletions(-) diff --git a/manifest b/manifest index 9519a7d2d6..061ce427ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sthe\sopenSubjournal()\sfunction\sin\spager.c.\s(CVS\s6130) -D 2009-01-07T10:52:56 +C Fix\sfor\s'truncate\sfile'\soperations\son\sin-memory\sdatabases.\s(CVS\s6131) +D 2009-01-07T15:18:21 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 05461a9b5803d5ad10c79f989801e9fd2cc3e592 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -142,12 +142,12 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60 F src/os_os2.c bed77dc26e3a95ce4a204936b9a1ca6fe612fcc5 F src/os_unix.c e6eacc7ec735ded605fefcbaf250058baa8feb12 F src/os_win.c 496e3ceb499aedc63622a89ef76f7af2dd902709 -F src/pager.c 1af21dadcfea3050b34ec54a5a173da368e13e10 -F src/pager.h 0793c5e4faed6c278037eb22b2434b318687d615 +F src/pager.c a799290e5cf9495f9991413ce84847011767b4d9 +F src/pager.h 9870acb2d653848d90d765d7cbf163496d6c8111 F src/parse.y 4d0e33a702dc3ea7b69d8ae1914b3fbd32e46057 F src/pcache.c 16dc8da6e6ba6250f8dfd9ee46036db1cbceedc6 F src/pcache.h f20c3e82dd6da622c3fe296170cb1801f9a2d75a -F src/pcache1.c 533b18aa2456b0f135e376289443e0a342e0c456 +F src/pcache1.c c0aa84ff69ea759fa944dbee9167a2463ab7c322 F src/pragma.c d6dfc47979623cf593d0db926daff49d214a8eb3 F src/prepare.c 51d11eb1eece7548038e7e33c33ee95dc7f82e2d F src/printf.c 9866a9a9c4a90f6d4147407f373df3fd5d5f9b6f @@ -164,7 +164,7 @@ F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76 F src/table.c 23db1e5f27c03160987c122a078b4bb51ef0b2f8 F src/tclsqlite.c 4415e1033bd3e92b05a6a9cde911ee4de3b82df9 F src/test1.c b193b8b80617bdb8297b25a87d00ee8d5a125d0d -F src/test2.c 15dbd8b99a05017eaafee9840c4c9dd63d26697a +F src/test2.c 724095fc69cb54b018aaa4463a56836c0b53a17a F src/test3.c 88a246b56b824275300e6c899634fbac1dc94b14 F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c F src/test5.c 162a1cea2105a2c460a3f39fa6919617b562a288 @@ -449,7 +449,7 @@ F test/mallocI.test 6e24fe6444bd2999ccc81f984977b44c0d6e5591 F test/mallocJ.test 44dfbbaca731cb933818ad300b4566265d652609 F test/malloc_common.tcl 984baeb6c6b185e798827d1187d426acc2bc4962 F test/manydb.test b3d3bc4c25657e7f68d157f031eb4db7b3df0d3c -F test/memdb.test a67bda4ff90a38f2b19f6c7f95aa7289e051d893 +F test/memdb.test a67c85a29d3187ac81d3628fcf9417d8a1be9ecb F test/memleak.test d2d2a1ff7105d32dc3fdf691458cf6cba58c7217 F test/memsubsys1.test bdc24a38d198679d777ca4efcc089fd3948567a6 F test/memsubsys2.test 72a731225997ad5e8df89fdbeae9224616b6aecc @@ -470,7 +470,7 @@ F test/nan.test c627d79b3d36ea892563fd67584b3e8a18f0618a F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test f5dd6b23e4dce828eb211649b600763c42a668df -F test/pager.test ef306077e282963664be71ac31832f4476e557ca +F test/pager.test 7e1788cce372ca0302a787d6b44821e269d10790 F test/pager2.test d4b7f6b70ff018b9995e622a32526b275f515042 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4 F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a @@ -692,7 +692,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e -P 3a4bb83235e9a79297e7d5d47ac7674c9df960bf -R 69328c811bc9ff20117c3aeb0aaeffc8 +P 04387ae10ab3be24c93497f4af6f48d6832f37eb +R cd03e1b03d4149ac01b912dddfeb4003 U danielk1977 -Z 001298448438542bba82708599474b69 +Z 242d556585dea66787f6cacf6e86748b diff --git a/manifest.uuid b/manifest.uuid index 308f578a87..14af207e6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04387ae10ab3be24c93497f4af6f48d6832f37eb \ No newline at end of file +83d1eafbde556f56969c6f285b6767d2c658dbfc \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 18800f82c8..130c8255ec 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.537 2009/01/07 10:52:56 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.538 2009/01/07 15:18:21 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -1086,6 +1086,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->setMaster = 0; pPager->needSync = 0; /* lruListSetFirstSynced(pPager); */ + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( !MEMDB ){ pPager->dbSizeValid = 0; } @@ -1462,10 +1463,11 @@ delmaster_out: /* -** Truncate the main file of the given pager to the number of pages -** indicated. Also truncate the cached representation of the file. +** If the main database file is open and an exclusive lock is held, +** truncate the main file of the given pager to the specified number +** of pages. ** -** Might might be the case that the file on disk is smaller than nPage. +** It might might be the case that the file on disk is smaller than nPage. ** This can happen, for example, if we are in the middle of a transaction ** which has extended the file size and the new pages are still all held ** in cache, then an INSERT or UPDATE does a statement rollback. Some @@ -1491,10 +1493,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ } } } - if( rc==SQLITE_OK ){ - pPager->dbSize = nPage; - sqlite3PcacheTruncate(pPager->pPCache, nPage); - } return rc; } @@ -1668,6 +1666,7 @@ static int pager_playback(Pager *pPager, int isHot){ if( rc!=SQLITE_OK ){ goto end_playback; } + pPager->dbSize = mxPg; } /* Copy original pages out of the journal and back into the database file. @@ -2349,41 +2348,12 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ return rc; } -/* -** Truncate the file to the number of pages specified. -** -** Unless an IO error occurs, this function is guaranteed to modify the -** database file itself. If an exclusive lock is not held when this function -** is called, one is obtained before truncating the file. -*/ -int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ - int rc = SQLITE_OK; - assert( pPager->state>=PAGER_SHARED ); - - sqlite3PagerPagecount(pPager, 0); - if( pPager->errCode ){ - rc = pPager->errCode; - }else if( nPagedbFileSize ){ - rc = syncJournal(pPager); - if( rc==SQLITE_OK ){ - /* Get an exclusive lock on the database before truncating. */ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } - if( rc==SQLITE_OK ){ - rc = pager_truncate(pPager, nPage); - } - } - - return rc; -} - #ifndef SQLITE_OMIT_AUTOVACUUM /* -** Truncate the in-memory database file image to nPage pages. Unlike -** sqlite3PagerTruncate(), this function does not actually modify the -** database file on disk. It just sets the internal state of the pager -** object so that the truncation will be done when the current -** transaction is committed. +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSizeValid ); @@ -2606,7 +2576,7 @@ static int pager_write_pagelist(PgHdr *pList){ } /* If there are dirty pages in the page cache with page numbers greater - ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to ** make the file smaller (presumably by auto-vacuum code). Do not write ** any such pages to the file. */ @@ -3963,13 +3933,6 @@ int sqlite3PagerCommitPhaseOne( } if( rc!=SQLITE_OK ) goto sync_exit; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPager->dbSizedbFileSize ){ - rc = sqlite3PagerTruncate(pPager, pPager->dbSize); - if( rc!=SQLITE_OK ) goto sync_exit; - } -#endif - /* Write all dirty pages to the database file */ pPg = sqlite3PcacheDirtyList(pPager->pPCache); rc = pager_write_pagelist(pPg); @@ -3986,6 +3949,12 @@ int sqlite3PagerCommitPhaseOne( } sqlite3PcacheCleanAll(pPager->pPCache); + if( pPager->dbSizedbFileSize ){ + assert( pPager->state>=PAGER_EXCLUSIVE ); + rc = pager_truncate(pPager, pPager->dbSize); + if( rc!=SQLITE_OK ) goto sync_exit; + } + /* Sync the database file. */ if( !pPager->noSync && !noSync ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); @@ -3993,8 +3962,6 @@ int sqlite3PagerCommitPhaseOne( IOTRACE(("DBSYNC %p\n", pPager)) pPager->state = PAGER_SYNCED; - }else if( MEMDB && pPager->dbSizedbFileSize ){ - rc = sqlite3PagerTruncate(pPager, pPager->dbSize); } sync_exit: diff --git a/src/pager.h b/src/pager.h index 8b7df843c4..9d466d89f4 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.92 2009/01/06 14:19:37 drh Exp $ +** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -88,7 +88,6 @@ int sqlite3PagerRef(DbPage*); int sqlite3PagerUnref(DbPage*); int sqlite3PagerWrite(DbPage*); int sqlite3PagerPagecount(Pager*, int*); -int sqlite3PagerTruncate(Pager*,Pgno); int sqlite3PagerBegin(DbPage*, int exFlag); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); int sqlite3PagerCommitPhaseTwo(Pager*); diff --git a/src/pcache1.c b/src/pcache1.c index 5f9dfa6796..1a8d6aad29 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -16,7 +16,7 @@ ** If the default page cache implementation is overriden, then neither of ** these two features are available. ** -** @(#) $Id: pcache1.c,v 1.6 2008/12/10 18:03:46 drh Exp $ +** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -47,6 +47,8 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ + + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ }; /* @@ -557,6 +559,9 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ } fetch_out: + if( pPage && iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } if( createFlag==1 ) sqlite3EndBenignMalloc(); pcache1LeaveMutex(); return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); @@ -632,6 +637,10 @@ static void pcache1Rekey( pPage->pNext = pCache->apHash[h]; pCache->apHash[h] = pPage; + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + pcache1LeaveMutex(); } @@ -645,7 +654,10 @@ static void pcache1Rekey( static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ PCache1 *pCache = (PCache1 *)p; pcache1EnterMutex(); - pcache1TruncateUnsafe(pCache, iLimit); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } pcache1LeaveMutex(); } diff --git a/src/test2.c b/src/test2.c index 5762714d10..21f2bffbba 100644 --- a/src/test2.c +++ b/src/test2.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test2.c,v 1.64 2009/01/02 18:10:42 drh Exp $ +** $Id: test2.c,v 1.65 2009/01/07 15:18:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -393,7 +393,6 @@ static int pager_truncate( const char **argv /* Text of each argument */ ){ Pager *pPager; - int rc; int pgno; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], @@ -402,11 +401,7 @@ static int pager_truncate( } pPager = sqlite3TestTextToPtr(argv[1]); if( Tcl_GetInt(interp, argv[2], &pgno) ) return TCL_ERROR; - rc = sqlite3PagerTruncate(pPager, pgno); - if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); - return TCL_ERROR; - } + sqlite3PagerTruncateImage(pPager, pgno); return TCL_OK; } diff --git a/test/memdb.test b/test/memdb.test index c1eb1152ce..deb8418afa 100644 --- a/test/memdb.test +++ b/test/memdb.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is in-memory database backend. # -# $Id: memdb.test,v 1.15 2006/01/30 22:48:44 drh Exp $ +# $Id: memdb.test,v 1.16 2009/01/07 15:18:21 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -411,6 +411,25 @@ do_test memdb-8.2 { } } 0 +# Test that auto-vacuum works with in-memory databases. +# +do_test memdb-9.1 { + db close + sqlite3 db test.db + db cache size 0 + execsql { + PRAGMA auto_vacuum = full; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(randstr(1000,1000)); + INSERT INTO t1 VALUES(randstr(1000,1000)); + INSERT INTO t1 VALUES(randstr(1000,1000)); + } + set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + execsql { DELETE FROM t1 } + set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] + expr {$memused2 + 2048 < $memused} +} {1} + } ;# ifcapable memorydb diff --git a/test/pager.test b/test/pager.test index 037112e5db..e227566618 100644 --- a/test/pager.test +++ b/test/pager.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # -# $Id: pager.test,v 1.33 2009/01/05 17:15:00 danielk1977 Exp $ +# $Id: pager.test,v 1.34 2009/01/07 15:18:21 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -413,7 +413,7 @@ do_test pager-4.6.1 { ifcapable memorydb { do_test pager-4.6.2 { set ::p2 [pager_open :memory: 10] - pager_truncate $::p2 5 + pager_truncate $::p2 0 } {} do_test pager-4.6.3 { set page1 [page_get $::p2 1] @@ -424,7 +424,7 @@ ifcapable memorydb { page_unref $p } page_unref $page1 - # pager_truncate $::p2 3 + pager_truncate $::p2 3 } {} do_test pager-4.6.4 { pager_close $::p2