From dddbcdcc6886be88e162d471f071ebc1f83b93fb Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Thu, 26 Apr 2007 14:42:34 +0000 Subject: [PATCH] Add largely untested code for the incremental vacuum function. (CVS 3876) FossilOrigin-Name: f6a6d2b8872c05089810b1e095f39011f3035408 --- manifest | 29 ++-- manifest.uuid | 2 +- src/btree.c | 350 ++++++++++++++++++++++++++----------------- src/btree.h | 8 +- src/build.c | 19 ++- src/parse.y | 6 +- src/pragma.c | 24 ++- src/sqliteInt.h | 3 +- src/vdbe.c | 20 ++- test/incrvacuum.test | 174 +++++++++++++++++++++ tool/mkkeywordhash.c | 11 +- 11 files changed, 482 insertions(+), 164 deletions(-) create mode 100644 test/incrvacuum.test diff --git a/manifest b/manifest index 468280e0d8..817c9f3c10 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\spager,\sload\sthe\scontent\sof\spages\swhich\swere\sinitialized\swith\nnoContent==1\sif\sthey\sare\ssubsequently\srequested\swith\snoContent==0.\s(CVS\s3875) -D 2007-04-26T12:11:28 +C Add\slargely\suntested\scode\sfor\sthe\sincremental\svacuum\sfunction.\s(CVS\s3876) +D 2007-04-26T14:42:35 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -59,9 +59,9 @@ F src/alter.c 2c79ec40f65e33deaf90ca493422c74586e481a3 F src/analyze.c 4bbf5ddf9680587c6d4917e02e378b6037be3651 F src/attach.c a16ada4a4654a0d126b8223ec9494ebb81bc5c3c F src/auth.c 902f4722661c796b97f007d9606bd7529c02597f -F src/btree.c 960bf64baa4d2bdb96019698e60d0b7763bf4e7e -F src/btree.h 9b2cc0d113c0bc2d37d244b9a394d56948c9acbf -F src/build.c 1880da163d9aa404016242b8b76d69907f682cd8 +F src/btree.c d08db3a8207bf884bd891829cab84c5e4cf18d99 +F src/btree.h 4c0b5855cef3e4e6627358aa69541d21a2015947 +F src/build.c 02e01ec7907c7d947ab3041fda0e81eaed05db42 F src/callback.c 6414ed32d55859d0f65067aa5b88d2da27b3af9e F src/complete.c 7d1a44be8f37de125fcafd3d3a018690b3799675 F src/date.c 94a6777df13d2aaacd19de080d9e8d3444364133 @@ -89,8 +89,8 @@ F src/os_win.c e94903c7dc1c0599c8ddce42efa0b6928068ddc5 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b F src/pager.c cd2770b0f8bd1900b46121009336e7ad03fb274f F src/pager.h d652ddf092d2318d00e41f8539760fe8e57c157c -F src/parse.y b6cfbadb6d5b21b5087d30698ee5af0ebb098767 -F src/pragma.c 3b992b5b2640d6ae25cef05aa6a42cd1d6c43234 +F src/parse.y a3940369e12c69c4968aa580cdc74cf73a664980 +F src/pragma.c 4fdefc03c3fd0ee87f8aad82bf80ba9bf1cdf416 F src/prepare.c 4cb9c9eb926e8baf5652ca4b4f2416f53f5b5370 F src/printf.c 0c6f40648770831341ac45ab32423a80b4c87f05 F src/random.c 6119474a6f6917f708c1dee25b9a8e519a620e88 @@ -99,7 +99,7 @@ F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96 F src/shell.c 3ae4654560e91220a95738a73d135d91d937cda1 F src/sqlite.h.in e429f66f9245c7f8675db24b230c950b8672ad1c F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890 -F src/sqliteInt.h 047af0e4c38bbb8652836f72adc9e9199c51a1ba +F src/sqliteInt.h 0b14d0eae083aafca0562d2261a404e5e5abc5f0 F src/table.c 6d0da66dde26ee75614ed8f584a1996467088d06 F src/tclsqlite.c ec69eb9ad56d03fbf7570ca1ca5ea947d1ec4b6f F src/test1.c 53b7eb5cba0012f592b5860f6ad3b5a3f887eb1e @@ -125,7 +125,7 @@ F src/update.c 3359041db390a8f856d67272f299600e2104f350 F src/utf.c e64a48bc21aa973eb622dd47da87d56a4cdcf528 F src/util.c b6344325378e75b9e18175d8b6aed1723d73dad9 F src/vacuum.c 8bd895d29e7074e78d4e80f948e35ddc9cf2beef -F src/vdbe.c 814dab208a156250bc5e77f827f4e0c8ad734820 +F src/vdbe.c a3cf3792fdbd382f756eb7eb50006b2f3f8d4283 F src/vdbe.h 0025259af1939fb264a545816c69e4b5b8d52691 F src/vdbeInt.h 4b19fd8febad3fd14c4c97adaefc06754d323132 F src/vdbeapi.c 245263aa2d70d87b1201753cddc881996f219843 @@ -238,6 +238,7 @@ F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51 F test/func.test 865febfd5b968f62b85c841c6a305b20346f7f44 F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d +F test/incrvacuum.test ee05edff95770f211fab28132291c175cd282e0c F test/index.test e65df12bed94b2903ee89987115e1578687e9266 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test f66718cd92ce1216819d47e6a156755e4b2c4ca1 @@ -394,7 +395,7 @@ F tool/lempar.c 8f998bf8d08e2123149c2cc5d0597cd5d5d1abdd F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133 F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf -F tool/mkkeywordhash.c c6f797bfc698803d2afbcbfb6b42f2239b074e29 +F tool/mkkeywordhash.c e119bdc04305adcada8856d73ad7d837c4ec123c F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x F tool/mksqlite3c.tcl 2d204fc271b2e2a2139e360527dd845385c4dffa F tool/mksqlite3internalh.tcl a85bb0c812db1a060e6e6dfab4e4c817f53d194b @@ -462,7 +463,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 9cb0ed6ee9827bc6884a0195044d5b6ad0de698e -R 654420a2ec3eb1f548ed11aa685883b0 -U drh -Z 352203b1a39e1bdcedb9d02c8002561c +P d0745a43b6e037d16e1ec38c7c4d961a80d1ef48 +R 9a772b637e3a7b43df139e14c80416c0 +U danielk1977 +Z d70505e165a313929d975eb44dd8ca1a diff --git a/manifest.uuid b/manifest.uuid index 18747468d4..d22a631dfe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0745a43b6e037d16e1ec38c7c4d961a80d1ef48 \ No newline at end of file +f6a6d2b8872c05089810b1e095f39011f3035408 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5512fa131c..412e48a05d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.358 2007/04/24 17:35:59 drh Exp $ +** $Id: btree.c,v 1.359 2007/04/26 14:42:35 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -336,7 +336,9 @@ struct BtShared { u8 minLeafFrac; /* Minimum leaf payload as % of total page size */ u8 pageSizeFixed; /* True if the page size can no longer be changed */ #ifndef SQLITE_OMIT_AUTOVACUUM - u8 autoVacuum; /* True if database supports auto-vacuum */ + u8 autoVacuum; /* True if auto-vacuum is enabled */ + u8 incrVacuum; /* True if incr-vacuum is enabled */ + Pgno nTrunc; /* Non-zero if the db will be truncated (incr vacuum) */ #endif u16 pageSize; /* Total number of bytes on a page */ u16 usableSize; /* Number of usable bytes on each page */ @@ -510,7 +512,6 @@ struct BtLock { #define unlockAllTables(a) #else - /* ** Query to see if btree handle p may obtain a lock of type eLock ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return @@ -1506,7 +1507,7 @@ int sqlite3BtreeOpen( ){ BtShared *pBt; /* Shared part of btree structure */ Btree *p; /* Handle to return */ - int rc; + int rc = SQLITE_OK; int nReserve; unsigned char zDbHeader[100]; #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) @@ -1569,22 +1570,15 @@ int sqlite3BtreeOpen( pBt = sqliteMalloc( sizeof(*pBt) ); if( pBt==0 ){ - *ppBtree = 0; - sqliteFree(p); - return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + goto btree_open_out; } rc = sqlite3PagerOpen(&pBt->pPager, zFilename, EXTRA_SIZE, flags); if( rc==SQLITE_OK ){ rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ - if( pBt->pPager ){ - sqlite3PagerClose(pBt->pPager); - } - sqliteFree(pBt); - sqliteFree(p); - *ppBtree = 0; - return rc; + goto btree_open_out; } p->pBt = pBt; @@ -1602,13 +1596,14 @@ int sqlite3BtreeOpen( pBt->minLeafFrac = 32; /* 12.5% */ #ifndef SQLITE_OMIT_AUTOVACUUM /* If the magic name ":memory:" will create an in-memory database, then - ** do not set the auto-vacuum flag, even if SQLITE_DEFAULT_AUTOVACUUM - ** is true. On the other hand, if SQLITE_OMIT_MEMORYDB has been defined, - ** then ":memory:" is just a regular file-name. Respect the auto-vacuum - ** default in this case. + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. */ if( zFilename && !isMemdb ){ - pBt->autoVacuum = SQLITE_DEFAULT_AUTOVACUUM; + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); } #endif nReserve = 0; @@ -1639,7 +1634,17 @@ int sqlite3BtreeOpen( #endif pBt->nRef = 1; *ppBtree = p; - return SQLITE_OK; + +btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ + sqlite3PagerClose(pBt->pPager); + } + sqliteFree(pBt); + sqliteFree(p); + *ppBtree = 0; + } + return rc; } /* @@ -1819,14 +1824,17 @@ int sqlite3BtreeGetReserve(Btree *p){ ** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. */ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ - BtShared *pBt = p->pBt;; #ifdef SQLITE_OMIT_AUTOVACUUM return SQLITE_READONLY; #else - if( pBt->pageSizeFixed ){ + BtShared *pBt = p->pBt; + int av = (autoVacuum?1:0); + int iv = (autoVacuum==BTREE_AUTOVACUUM_INCR?1:0); + if( pBt->pageSizeFixed && av!=pBt->autoVacuum ){ return SQLITE_READONLY; } - pBt->autoVacuum = (autoVacuum?1:0); + pBt->autoVacuum = av; + pBt->incrVacuum = iv; return SQLITE_OK; #endif } @@ -1837,9 +1845,13 @@ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ */ int sqlite3BtreeGetAutoVacuum(Btree *p){ #ifdef SQLITE_OMIT_AUTOVACUUM - return 0; + return BTREE_AUTOVACUUM_NONE; #else - return p->pBt->autoVacuum; + return ( + (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: + (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: + BTREE_AUTOVACUUM_INCR + ); #endif } @@ -1998,9 +2010,8 @@ static int newDatabase(BtShared *pBt){ zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); pBt->pageSizeFixed = 1; #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - put4byte(&data[36 + 4*4], 1); - } + assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); + put4byte(&data[36 + 4*4], pBt->autoVacuum); #endif return SQLITE_OK; } @@ -2285,9 +2296,121 @@ static int relocatePage( return rc; } -/* Forward declaration required by autoVacuumCommit(). */ +/* Forward declaration required by incrVacuumStep(). */ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); +/* +** Perform a single step of an incremental-vacuum. If successful, +** return SQLITE_OK. If there is no work to do (and therefore no +** point in calling this function again), return SQLITE_DONE. +** +** More specificly, this function attempts to re-organize the +** database so that the last page of the file currently in use +** is no longer in use. +** +** If the nFin parameter is non-zero, the implementation assumes +** that the caller will keep calling incrVacuumStep() until +** it returns SQLITE_DONE or an error, and that nFin is the +** number of pages the database file will contain after this +** process is complete. +*/ +static int incrVacuumStep(BtShared *pBt, Pgno nFin){ + Pgno iLastPg; /* Last page in the database */ + Pgno nFreeList; /* Number of pages still on the free-list */ + + iLastPg = pBt->nTrunc; + if( iLastPg==0 ){ + iLastPg = sqlite3PagerPagecount(pBt->pPager); + } + + if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ + int rc; + u8 eType; + Pgno iPtrPage; + + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 || nFin==iLastPg ){ + return SQLITE_DONE; + } + + rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); + if( rc!=SQLITE_OK ){ + return rc; + } + if( eType==PTRMAP_ROOTPAGE ){ + return SQLITE_CORRUPT_BKPT; + } + + if( eType==PTRMAP_FREEPAGE ){ + if( nFin==0 ){ + /* Remove the page from the files free-list. This is not required + ** if nFin is non-zero. In this case, the free-list will be + ** truncated to zero after this function returns, so it doesn't + ** matter if it still contains some garbage entries. + */ + Pgno iFreePg; + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( iFreePg==iLastPg ); + releasePage(pFreePg); + } + } else { + Pgno iFreePg; /* Index of free page to move pLastPg to */ + MemPage *pLastPg; + + rc = getPage(pBt, iLastPg, &pLastPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + do { + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0); + if( rc!=SQLITE_OK ){ + releasePage(pLastPg); + return rc; + } + releasePage(pFreePg); + }while( nFin!=0 && iFreePg>nFin ); + assert( iFreePgnTrunc = iLastPg - 1; + while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){ + pBt->nTrunc--; + } + return SQLITE_OK; +} + +/* +** A write-transaction must be opened before calling this function. +** It performs a single unit of work towards an incremental vacuum. +** +** If the incremental vacuum is finished after this function has run, +** SQLITE_DONE is returned. If it is not finished, but no error occured, +** SQLITE_OK is returned. Otherwise an SQLite error code. +*/ +int sqlite3BtreeIncrVacuum(Btree *p){ + BtShared *pBt = p->pBt; + + assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); + if( !pBt->autoVacuum ){ + return SQLITE_DONE; + } + + return incrVacuumStep(p->pBt, 0); +} + /* ** This routine is called prior to sqlite3PagerCommit when a transaction ** is commited for an auto-vacuum database. @@ -2298,135 +2421,65 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** pages are in use. */ static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){ + int rc = SQLITE_OK; Pager *pPager = pBt->pPager; - Pgno nFreeList; /* Number of pages remaining on the free-list. */ - int nPtrMap; /* Number of pointer-map pages deallocated */ - Pgno origSize; /* Pages in the database file */ - Pgno finSize; /* Pages in the database file after truncation */ - int rc; /* Return code */ - u8 eType; - int pgsz = pBt->pageSize; /* Page size for this database */ - Pgno iDbPage; /* The database page to move */ - MemPage *pDbMemPage = 0; /* "" */ - Pgno iPtrPage; /* The page that contains a pointer to iDbPage */ - Pgno iFreePage; /* The free-list page to move iDbPage to */ - MemPage *pFreeMemPage = 0; /* "" */ - #ifndef NDEBUG int nRef = sqlite3PagerRefcount(pPager); #endif - assert( pBt->autoVacuum ); if( PTRMAP_ISPAGE(pBt, sqlite3PagerPagecount(pPager)) ){ return SQLITE_CORRUPT_BKPT; } - /* Figure out how many free-pages are in the database. If there are no - ** free pages, then auto-vacuum is a no-op. - */ - nFreeList = get4byte(&pBt->pPage1->aData[36]); - if( nFreeList==0 ){ - *pnTrunc = 0; - return SQLITE_OK; - } + assert(pBt->autoVacuum); + if( !pBt->incrVacuum ){ + Pgno nFin = 0; - /* This block figures out how many pages there are in the database - ** now (variable origSize), and how many there will be after the - ** truncation (variable finSize). - ** - ** The final size is the original size, less the number of free pages - ** in the database, less any pointer-map pages that will no longer - ** be required, less 1 if the pending-byte page was part of the database - ** but is not after the truncation. - **/ - origSize = sqlite3PagerPagecount(pPager); - if( origSize==PENDING_BYTE_PAGE(pBt) ){ - origSize--; - } - nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pBt, origSize)+pgsz/5)/(pgsz/5); - finSize = origSize - nFreeList - nPtrMap; - if( origSize>PENDING_BYTE_PAGE(pBt) && finSize<=PENDING_BYTE_PAGE(pBt) ){ - finSize--; - } - while( PTRMAP_ISPAGE(pBt, finSize) || finSize==PENDING_BYTE_PAGE(pBt) ){ - finSize--; - } - TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize)); - - /* Variable 'finSize' will be the size of the file in pages after - ** the auto-vacuum has completed (the current file size minus the number - ** of pages on the free list). Loop through the pages that lie beyond - ** this mark, and if they are not already on the free list, move them - ** to a free page earlier in the file (somewhere before finSize). - */ - for( iDbPage=finSize+1; iDbPage<=origSize; iDbPage++ ){ - /* If iDbPage is a pointer map page, or the pending-byte page, skip it. */ - if( PTRMAP_ISPAGE(pBt, iDbPage) || iDbPage==PENDING_BYTE_PAGE(pBt) ){ - continue; - } - - rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage); - if( rc!=SQLITE_OK ) goto autovacuum_out; - if( eType==PTRMAP_ROOTPAGE ){ - rc = SQLITE_CORRUPT_BKPT; - goto autovacuum_out; - } - - /* If iDbPage is free, do not swap it. */ - if( eType==PTRMAP_FREEPAGE ){ - continue; - } - rc = getPage(pBt, iDbPage, &pDbMemPage, 0); - if( rc!=SQLITE_OK ) goto autovacuum_out; - - /* Find the next page in the free-list that is not already at the end - ** of the file. A page can be pulled off the free list using the - ** allocateBtreePage() routine. - */ - do{ - if( pFreeMemPage ){ - releasePage(pFreeMemPage); - pFreeMemPage = 0; + if( pBt->nTrunc==0 ){ + Pgno nFree; + Pgno nPtrmap; + const int pgsz = pBt->pageSize; + Pgno nOrig = sqlite3PagerPagecount(pBt->pPager); + if( nOrig==PENDING_BYTE_PAGE(pBt) ){ + nOrig--; } - rc = allocateBtreePage(pBt, &pFreeMemPage, &iFreePage, 0, 0); - if( rc!=SQLITE_OK ){ - releasePage(pDbMemPage); - goto autovacuum_out; + nFree = get4byte(&pBt->pPage1->aData[36]); + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5); + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ + nFin--; } - assert( iFreePage<=origSize ); - }while( iFreePage>finSize ); - releasePage(pFreeMemPage); - pFreeMemPage = 0; + while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + } - /* Relocate the page into the body of the file. Note that although the - ** page has moved within the database file, the pDbMemPage pointer - ** remains valid. This means that this function can run without - ** invalidating cursors open on the btree. This is important in - ** shared-cache mode. - */ - rc = relocatePage(pBt, pDbMemPage, eType, iPtrPage, iFreePage); - releasePage(pDbMemPage); - if( rc!=SQLITE_OK ) goto autovacuum_out; + while( rc==SQLITE_OK ){ + rc = incrVacuumStep(pBt, nFin); + } + if( rc==SQLITE_DONE ){ + assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc); + rc = SQLITE_OK; + if( pBt->nTrunc ){ + sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + pBt->nTrunc = nFin; + } + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); + } } - /* The entire free-list has been swapped to the end of the file. So - ** truncate the database file to finSize pages and consider the - ** free-list empty. - */ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - if( rc!=SQLITE_OK ) goto autovacuum_out; - put4byte(&pBt->pPage1->aData[32], 0); - put4byte(&pBt->pPage1->aData[36], 0); - *pnTrunc = finSize; - assert( finSize!=PENDING_BYTE_PAGE(pBt) ); - -autovacuum_out: + if( rc==SQLITE_OK ){ + *pnTrunc = pBt->nTrunc; + pBt->nTrunc = 0; + } assert( nRef==sqlite3PagerRefcount(pPager) ); - if( rc!=SQLITE_OK ){ - sqlite3PagerRollback(pPager); - } return rc; } + #endif /* @@ -2615,6 +2668,10 @@ int sqlite3BtreeRollback(Btree *p){ if( p->inTrans==TRANS_WRITE ){ int rc2; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->nTrunc = 0; +#endif + assert( TRANS_WRITE==pBt->inTransaction ); rc2 = sqlite3PagerRollback(pBt->pPager); if( rc2!=SQLITE_OK ){ @@ -3854,6 +3911,16 @@ static int allocateBtreePage( *pPgno = sqlite3PagerPagecount(pBt->pPager) + 1; #ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->nTrunc ){ + /* An incr-vacuum has already run within this transaction. So the + ** page to allocate is not from the physical end of the file, but + ** at pBt->nTrunc. + */ + *pPgno = pBt->nTrunc+1; + if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ + (*pPgno)++; + } + } if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){ /* If *pPgno refers to a pointer-map page, allocate two new pages ** at the end of the file instead of one. The first allocated page @@ -3863,6 +3930,9 @@ static int allocateBtreePage( assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); (*pPgno)++; } + if( pBt->nTrunc ){ + pBt->nTrunc = *pPgno; + } #endif assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); diff --git a/src/btree.h b/src/btree.h index 50b5317b62..0d591ef46d 100644 --- a/src/btree.h +++ b/src/btree.h @@ -13,7 +13,7 @@ ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. ** -** @(#) $Id: btree.h,v 1.74 2007/03/30 14:06:34 drh Exp $ +** @(#) $Id: btree.h,v 1.75 2007/04/26 14:42:36 danielk1977 Exp $ */ #ifndef _BTREE_H_ #define _BTREE_H_ @@ -31,6 +31,10 @@ #define SQLITE_DEFAULT_AUTOVACUUM 0 #endif +#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ +#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ +#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ + /* ** Forward declarations of structure */ @@ -87,6 +91,8 @@ const char *sqlite3BtreeGetDirname(Btree *); const char *sqlite3BtreeGetJournalname(Btree *); int sqlite3BtreeCopyFile(Btree *, Btree *); +int sqlite3BtreeIncrVacuum(Btree *); + /* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR ** of the following flags: */ diff --git a/src/build.c b/src/build.c index 41bb82b019..d3ecf4f673 100644 --- a/src/build.c +++ b/src/build.c @@ -22,7 +22,7 @@ ** COMMIT ** ROLLBACK ** -** $Id: build.c,v 1.421 2007/04/18 14:47:24 danielk1977 Exp $ +** $Id: build.c,v 1.422 2007/04/26 14:42:36 danielk1977 Exp $ */ #include "sqliteInt.h" #include @@ -3349,3 +3349,20 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ } return pKey; } + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** This is called to compile a statement of the form "INCREMENTAL VACUUM". +*/ +void sqlite3IncrVacuum(Parse *pParse){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + int addr; + sqlite3BeginWriteOperation(pParse, 0, 0); + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp(v, OP_IncrVacuum, 0, addr+3); + sqlite3VdbeAddOp(v, OP_Callback, 0, 0); + sqlite3VdbeAddOp(v, OP_Goto, 0, addr); + } +} +#endif /* #ifndef SQLITE_OMIT_AUTOVACUUM */ diff --git a/src/parse.y b/src/parse.y index ae71956b61..c676270b76 100644 --- a/src/parse.y +++ b/src/parse.y @@ -14,7 +14,7 @@ ** the parser. Lemon will also generate a header file containing ** numeric codes for all of the tokens. ** -** @(#) $Id: parse.y,v 1.218 2007/04/06 15:02:14 drh Exp $ +** @(#) $Id: parse.y,v 1.219 2007/04/26 14:42:36 danielk1977 Exp $ */ // All token codes are small integers with #defines that begin with "TK_" @@ -903,6 +903,10 @@ cmd ::= VACUUM nm. {sqlite3Vacuum(pParse);} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM +%ifndef SQLITE_OMIT_AUTOVACUUM +cmd ::= INCREMENTAL VACUUM. {sqlite3IncrVacuum(pParse);} +%endif + ///////////////////////////// The PRAGMA command ///////////////////////////// // %ifndef SQLITE_OMIT_PRAGMA diff --git a/src/pragma.c b/src/pragma.c index ff7b48655c..68fd9f18a5 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code used to implement the PRAGMA command. ** -** $Id: pragma.c,v 1.132 2007/03/30 17:11:13 danielk1977 Exp $ +** $Id: pragma.c,v 1.133 2007/04/26 14:42:36 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -73,6 +73,23 @@ static int getLockingMode(const char *z){ return PAGER_LOCKINGMODE_QUERY; } +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Interpret the given string as an auto-vacuum mode value. +** +** The following strings, "none", "full" and "incremental" are +** acceptable, as are their numeric equivalents: 0, 1 and 2 respectively. +*/ +static int getAutoVacuum(const char *z){ + int i; + if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE; + if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL; + if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR; + i = atoi(z); + return ((i>=0&&i<=2)?i:0); +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + #ifndef SQLITE_OMIT_PAGER_PRAGMAS /* ** Interpret the given string as a temp db location. Return 1 for file @@ -389,7 +406,10 @@ void sqlite3Pragma( pBt ? sqlite3BtreeGetAutoVacuum(pBt) : SQLITE_DEFAULT_AUTOVACUUM; returnSingleInt(pParse, "auto_vacuum", auto_vacuum); }else{ - sqlite3BtreeSetAutoVacuum(pBt, getBoolean(zRight)); + int eAuto = getAutoVacuum(zRight); + if( eAuto>=0 ){ + sqlite3BtreeSetAutoVacuum(pBt, eAuto); + } } }else #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 039d22f953..8721cd53ef 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.552 2007/04/16 15:06:25 danielk1977 Exp $ +** @(#) $Id: sqliteInt.h,v 1.553 2007/04/26 14:42:36 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -1913,6 +1913,7 @@ int sqlite3VtabBegin(sqlite3 *, sqlite3_vtab *); FuncDef *sqlite3VtabOverloadFunction(FuncDef*, int nArg, Expr*); void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**); int sqlite3Reprepare(Vdbe*); +void sqlite3IncrVacuum(Parse *pParse); #ifdef SQLITE_SSE #include "sseInt.h" diff --git a/src/vdbe.c b/src/vdbe.c index 06232046d7..eeb0847fd3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -43,7 +43,7 @@ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. ** -** $Id: vdbe.c,v 1.601 2007/04/18 16:45:24 drh Exp $ +** $Id: vdbe.c,v 1.602 2007/04/26 14:42:36 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -4554,6 +4554,24 @@ case OP_Vacuum: { /* no-push */ } #endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) +/* Opcode: IncrVacuum * P2 * +** +** Perform a single step of the incremental vacuum procedure on +** the main database. If the vacuum has finished, jump to instruction +** P2. Otherwise, fall through to the next instruction. +*/ +case OP_IncrVacuum: { /* no-push */ + Btree *pBt = db->aDb[0].pBt; + rc = sqlite3BtreeIncrVacuum(pBt); + if( rc==SQLITE_DONE ){ + pc = pOp->p2 - 1; + rc = SQLITE_OK; + } + break; +} +#endif + /* Opcode: Expire P1 * * ** ** Cause precompiled statements to become expired. An expired statement diff --git a/test/incrvacuum.test b/test/incrvacuum.test new file mode 100644 index 0000000000..9ff25d3dcd --- /dev/null +++ b/test/incrvacuum.test @@ -0,0 +1,174 @@ +# 2007 April 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the incremental vacuum feature. +# +# $Id: incrvacuum.test,v 1.1 2007/04/26 14:42:36 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build of the library does not support auto-vacuum, omit this +# whole file. +ifcapable {!autovacuum || !pragma} { + finish_test + return +} + +#--------------------------------------------------------------------- +# Test the pragma on an empty database. +# +do_test incrvacuum-1.1 { + execsql { + pragma auto_vacuum; + } +} {0} +do_test incrvacuum-1.2 { + execsql { + pragma auto_vacuum = 'full'; + pragma auto_vacuum; + } +} {1} +do_test incrvacuum-1.3 { + execsql { + pragma auto_vacuum = 'incremental'; + pragma auto_vacuum; + } +} {2} +do_test incrvacuum-1.4 { + execsql { + pragma auto_vacuum = 'invalid'; + pragma auto_vacuum; + } +} {0} +do_test incrvacuum-1.5 { + execsql { + pragma auto_vacuum = 1; + pragma auto_vacuum; + } +} {1} +do_test incrvacuum-1.6 { + execsql { + pragma auto_vacuum = '2'; + pragma auto_vacuum; + } +} {2} +do_test incrvacuum-1.7 { + execsql { + pragma auto_vacuum = 5; + pragma auto_vacuum; + } +} {0} + +#--------------------------------------------------------------------- +# Test the pragma on a non-empty database. It is possible to toggle +# the connection between "full" and "incremental" mode, but not to +# change from either of these to "none", or from "none" to "full" or +# "incremental". +# +do_test incrvacuum-2.1 { + execsql { + pragma auto_vacuum = 1; + CREATE TABLE abc(a, b, c); + } +} {} +do_test incrvacuum-2.2 { + execsql { + pragma auto_vacuum = 'none'; + pragma auto_vacuum; + } +} {1} +do_test incrvacuum-2.3 { + execsql { + pragma auto_vacuum = 'incremental'; + pragma auto_vacuum; + } +} {2} +do_test incrvacuum-2.4 { + execsql { + pragma auto_vacuum = 'full'; + pragma auto_vacuum; + } +} {1} + +#--------------------------------------------------------------------- +# Test that when the auto_vacuum mode is "incremental", the database +# does not shrink when pages are removed from it. But it does if +# the mode is set to "full". +# +do_test incrvacuum-3.1 { + execsql { + pragma auto_vacuum; + } +} {1} +do_test incrvacuum-3.2 { + set ::str [string repeat 1234567890 110] + execsql { + PRAGMA auto_vacuum = 2; + BEGIN; + CREATE TABLE tbl2(str); + INSERT INTO tbl2 VALUES($::str); + COMMIT; + } + # 5 pages: + # + # 1 -> database header + # 2 -> first back-pointer page + # 3 -> table abc + # 4 -> table tbl2 + # 5 -> table tbl2 overflow page. + # + expr {[file size test.db] / 1024} +} {5} +do_test incrvacuum-3.3 { + execsql { + DROP TABLE abc; + DELETE FROM tbl2; + } + expr {[file size test.db] / 1024} +} {5} +do_test incrvacuum-3.4 { + execsql { + PRAGMA auto_vacuum = 1; + INSERT INTO tbl2 VALUES('hello world'); + } + expr {[file size test.db] / 1024} +} {3} + +#--------------------------------------------------------------------- +# Try to run a simple incremental vacuum. +# +do_test incrvacuum-4.1 { + set ::str [string repeat 1234567890 110] + execsql { + PRAGMA auto_vacuum = 2; + INSERT INTO tbl2 VALUES($::str); + CREATE TABLE tbl1(a, b, c); + } + expr {[file size test.db] / 1024} +} {5} +do_test incrvacuum-4.2 { + execsql { + DELETE FROM tbl2; + DROP TABLE tbl1; + } + expr {[file size test.db] / 1024} +} {5} +do_test incrvacuum-4.3 { + set ::nStep 0 + db eval {INCREMENTAL VACUUM} { + incr ::nStep + } + list [expr {[file size test.db] / 1024}] $::nStep +} {3 2} + +finish_test + diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index 8e8cb96329..d1fd431f33 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -15,7 +15,7 @@ static const char zHdr[] = "**\n" "** The code in this file has been automatically generated by\n" "**\n" - "** $Header: /home/drh/sqlite/trans/cvs/sqlite/sqlite/tool/mkkeywordhash.c,v 1.27 2007/04/06 11:26:00 drh Exp $\n" + "** $Header: /home/drh/sqlite/trans/cvs/sqlite/sqlite/tool/mkkeywordhash.c,v 1.28 2007/04/26 14:42:36 danielk1977 Exp $\n" "**\n" "** The code in this file implements a function that determines whether\n" "** or not a given identifier is really an SQL keyword. The same thing\n" @@ -114,7 +114,8 @@ struct Keyword { #else # define TRIGGER 0x00002000 #endif -#if defined(SQLITE_OMIT_VACUUM) || defined(SQLITE_OMIT_ATTACH) +#if defined(SQLITE_OMIT_AUTOVACUUM) && \ + (defined(SQLITE_OMIT_VACUUM) || defined(SQLITE_OMIT_ATTACH)) # define VACUUM 0 #else # define VACUUM 0x00004000 @@ -129,6 +130,11 @@ struct Keyword { #else # define VTAB 0x00010000 #endif +#ifdef SQLITE_OMIT_AUTOVACUUM +# define AUTOVACUUM 0 +#else +# define AUTOVACUUM 0x00020000 +#endif /* ** These are the keywords @@ -192,6 +198,7 @@ static Keyword aKeywordTable[] = { { "IGNORE", "TK_IGNORE", CONFLICT|TRIGGER }, { "IMMEDIATE", "TK_IMMEDIATE", ALWAYS }, { "IN", "TK_IN", ALWAYS }, + { "INCREMENTAL", "TK_INCREMENTAL", AUTOVACUUM }, { "INDEX", "TK_INDEX", ALWAYS }, { "INITIALLY", "TK_INITIALLY", FKEY }, { "INNER", "TK_JOIN_KW", ALWAYS },