diff --git a/manifest b/manifest index 5ac026a2ad..9eda2d1148 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\ssupport.html\spage.\s(CVS\s2036) -D 2004-11-01T16:03:12 +C Code\sto\sauto-vacuum\sthe\sdatabase\sif\sall\sroot\spages\shappen\sto\sbe\sin\sthe\sright\splace.\sNot\sactive\sby\sdefault\sand\slargely\suntested.\s(CVS\s2037) +D 2004-11-02T12:56:41 F Makefile.in 9e90c685d69f09039015a6b1f3b0a48e9738c9e5 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1 @@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea -F src/btree.c 231a0e7a00b96bafd4a7a0c774eece1510aec2fd +F src/btree.c 92713a104c11abe0826a33590f1d4147332fd9a5 F src/btree.h 94dfec0a1722d33359b23e7e310f2b64ffedf029 F src/build.c bb896c5f85ab749d17ae5d730235134c12c08033 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad @@ -52,8 +52,8 @@ F src/os_unix.c 5824b22ba41fe9d514ef9169aac1b5fde73af229 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13 F src/os_win.c 9482dfc92f289b68205bb2c9315757c7e3946bfb F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c 6e19f9a64a9fae60bcf00140cecb5981765f3d95 -F src/pager.h 774d1973acbda341827d21b0da0150575d69f7d9 +F src/pager.c 6b00c0d5aac601b9f556b8fdba25e69438245f1a +F src/pager.h cbe4ba356d9dd3f30260f322b3dc77408164df14 F src/parse.y 08f4971f89e651f47b3f83fe7369c7edde254331 F src/pragma.c 44e192eb5928157bdb015926f858a7c6e3ef6c98 F src/printf.c 7a92adc00b758cd5ce087dae80181a8bbdb70ed2 @@ -87,6 +87,7 @@ F test/attach.test 6ad560eb3e77751a4faecd77da09deac9e38cc41 F test/attach2.test f7795123d3051ace1672b6d23973da6435de3745 F test/attach3.test 6d060986ff004ebb89e1876a331d96c6bb62269e F test/auth.test 1cc252d9e7b3bdc1314199cbf3a0d3c5ed026c21 +F test/autovacuum.test 77eec318b1be7764b8dcb3198c035edc30cd319f F test/bigfile.test d3744a8821ce9abb8697f2826a3e3d22b719e89f F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 F test/bind.test a8682ba41433b93bb36a4213a43f282ca9aec5a9 @@ -157,7 +158,7 @@ F test/pagesize.test 56d11f4d6df9949d646bf87da1d6d995ed37cd78 F test/pragma.test 66a66b7f3b273b93325c9a5794acb418f52fdcbf F test/printf.test 92ba4c510b4fc61120ffa4a01820446ed917ae57 F test/progress.test 5ddba78cb6011fba36093973cfb3ac473b8fb96a x -F test/quick.test 212a9cd4c40c72c7c4780fef1c2fbe5d1cb34ce6 +F test/quick.test 2dca186ebd5c418a7699944ba3b5e437d765eddd F test/quote.test 6d75cf635d93ba2484dc9cb378d88cbae9dc2c62 F test/rollback.test 4097328d44510277244ef4fa51b22b2f11d7ef4c F test/rowid.test b3d059f5c8d8874fa1c31030e0636f67405d20ea @@ -251,7 +252,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c -P bebd967f3627220c3ce0352c8ca9c7c17b722ce6 -R 05340acf98e3d142b518cfefba2b2bd8 -U drh -Z ee6b8d03ade5cb666f8adaa6773746cd +P 5515accee348c6364cd58903a19029519797e123 +R d09732475e8e7fd32816b925d21968c5 +U danielk1977 +Z 3ba6d76f77fb940670c05d63aad699f9 diff --git a/manifest.uuid b/manifest.uuid index 1a61e7be30..2d1072716a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5515accee348c6364cd58903a19029519797e123 \ No newline at end of file +d12481f09cbe51c7ea499bc22afec5de3af14ad4 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 031bf635cd..b7d804c580 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.195 2004/10/31 16:25:43 danielk1977 Exp $ +** $Id: btree.c,v 1.196 2004/11/02 12:56:41 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -414,23 +414,42 @@ static void put4byte(unsigned char *p, u32 v){ #define PTRMAP_PTROFFSET(pgsz, pgno) (((pgno-2)%(pgsz/5+1)-1)*5) /* -** The first byte of each 5-byte pointer map entry identifies the type -** of page that the following 4-byte page number refers to (either a -** regular btree page or an overflow page). +** The pointer map is a lookup table that contains an entry for each database +** page in the file except for page 1. In this context 'database page' refers +** to any page that is not part of the pointer map itself. Each pointer map +** entry consists of a single byte 'type' and a 4 byte page number. The +** PTRMAP_XXX identifiers below are the valid types. The interpretation +** of the page-number depends on the type, as follows: ** -** If the type is PTRMAP_OVERFLOW, then the page is an overflow page. -** In this case the pointer is always the first 4 bytes of the page. +** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not +** used in this case. ** -** If the type is PTRMAP_BTREE, then the page is a btree page. In this -** case the pointer may be a 'left-pointer' (stored following a cell-header), -** a pointer to an overflow page (stored after a cell's data payload), -** or the 'right pointer' of a btree page. +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** is not used in this case. +** +** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** overflow pages. The page number identifies the page that +** contains the cell with a pointer to this overflow page. +** +** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of +** overflow pages. The page-number identifies the previous +** page in the overflow page list. +** +** PTRMAP_BTREE: The database page is a non-root btree page. The page number +** identifies the parent page in the btree. */ -#define PTRMAP_BTREE 1 -#define PTRMAP_OVERFLOW 2 +#define PTRMAP_ROOTPAGE 1 +#define PTRMAP_FREEPAGE 2 +#define PTRMAP_OVERFLOW1 3 +#define PTRMAP_OVERFLOW2 4 +#define PTRMAP_BTREE 5 /* ** Write an entry into the pointer map. +** +** This routine updates the pointer map entry for page number 'key' +** so that it maps to type 'eType' and parent page number 'pgno'. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. */ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ u8 *pPtrmap; /* The pointer map page */ @@ -440,7 +459,7 @@ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ iPtrmap = PTRMAP_PAGENO(pBt->pageSize, key); rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap); - if( rc!=0 ){ + if( rc!=SQLITE_OK ){ return rc; } offset = PTRMAP_PTROFFSET(pBt->pageSize, key); @@ -460,6 +479,10 @@ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ /* ** Read an entry from the pointer map. +** +** This routine retrieves the pointer map entry for page 'key', writing +** the type and parent page number to *pEType and *pPgno respectively. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. */ static int ptrmapGet(Btree *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ int iPtrmap; /* Pointer map page index */ @@ -474,8 +497,8 @@ static int ptrmapGet(Btree *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ } offset = PTRMAP_PTROFFSET(pBt->pageSize, key); - *pEType = pPtrmap[offset]; - *pPgno = get4byte(&pPtrmap[offset+1]); + if( pEType ) *pEType = pPtrmap[offset]; + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); sqlite3pager_unref(pPtrmap); return SQLITE_OK; @@ -1181,7 +1204,9 @@ int sqlite3BtreeOpen( *ppBtree = pBt; #ifdef SQLITE_AUTOVACUUM /* Note: This is temporary code for use during development of auto-vacuum. */ - pBt->autoVacuum = 1; + if( zFilename && 0!=strcmp(zFilename, ":memory:") ){ + pBt->autoVacuum = 1; + } #endif return SQLITE_OK; } @@ -1453,6 +1478,272 @@ int sqlite3BtreeBeginTrans(Btree *pBt, int wrflag){ return rc; } +/* +** The TRACE macro will print high-level status information about the +** btree operation when the global variable sqlite3_btree_trace is +** enabled. +*/ +#if SQLITE_TEST +# define TRACE(X) if( sqlite3_btree_trace )\ + { sqlite3DebugPrintf X; fflush(stdout); } +#else +# define TRACE(X) +#endif +int sqlite3_btree_trace=0; /* True to enable tracing */ + +#ifndef SQLITE_OMIT_AUTOVACUUM + +/* +** Set the pointer-map entries for all children of page pPage. Also, if +** pPage contains cells that point to overflow pages, set the pointer +** map entries for the overflow pages as well. +*/ +static int setChildPtrmaps(MemPage *pPage){ + int i; /* Counter variable */ + int nCell; /* Number of cells in page pPage */ + int rc = SQLITE_OK; /* Return code */ + Btree *pBt = pPage->pBt; + int isInitOrig = pPage->isInit; + Pgno pgno = pPage->pgno; + + initPage(pPage, 0); + nCell = pPage->nCell; + + for(i=0; ileaf ){ + Pgno childPgno = get4byte(pCell); + rc = ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno); + if( rc!=SQLITE_OK ) goto set_child_ptrmaps_out; + } + } + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + rc = ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno); + } + +set_child_ptrmaps_out: + pPage->isInit = isInitOrig; + return rc; +} + +/* +** Somewhere on pPage, which is guarenteed to be a btree page, not an overflow +** page, is a pointer to page iFrom. Modify this pointer so that it points to +** iTo. Parameter eType describes the type of pointer to be modified, as +** follows: +** +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** page of pPage. +** +** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow +** page pointed to by one of the cells on pPage. +** +** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next +** overflow page in the list. +*/ +static void modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + + if( eType==PTRMAP_OVERFLOW2 ){ + assert( get4byte(pPage->aData)==iFrom ); + put4byte(pPage->aData, iFrom); + }else{ + int isInitOrig = pPage->isInit; + int i; + int nCell; + + initPage(pPage, 0); + nCell = pPage->nCell; + + /* assert( !pPage->leaf && eType==PTRMAP_BTREE ); */ + + for(i=0; iaData[pPage->hdrOffset+8])==iFrom ); + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); + } + + pPage->isInit = isInitOrig; + } +} + +/* Forward declaration required by autoVacuumCommit(). */ +static int allocatePage(Btree *, MemPage **, Pgno *, Pgno); + +/* +** This routine is called prior to sqlite3pager_commit when a transaction +** is commited for an auto-vacuum database. +*/ +static int autoVacuumCommit(Btree *pBt){ + Pager *pPager = pBt->pPager; + Pgno nFreeList; /* Number of pages remaining on the free-list. */ + Pgno origDbSize; /* Pages in the database file */ + Pgno finDbSize; /* Pages in the database file after truncation */ + int i; /* Counter variable */ + int rc; /* Return code */ + u8 eType; + + Pgno iDbPage; /* The database page to move */ + u8 *pDbPage = 0; /* "" */ + MemPage *pDbMemPage = 0; /* "" */ + Pgno iPtrPage; /* The page that contains a pointer to iDbPage */ + MemPage *pPtrMemPage = 0; /* "" */ + Pgno iFreePage; /* The free-list page to move iDbPage to */ + MemPage *pFreeMemPage = 0; /* "" */ + +#ifndef NDEBUG + int nRef = *sqlite3pager_stats(pPager); +#endif + + assert( pBt->autoVacuum ); + + /* Figure out how many free-pages are in the database. If there are no + ** free pages, then auto-vacuum is a no-op. + */ + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 ) return SQLITE_OK; + + origDbSize = sqlite3pager_pagecount(pPager); + finDbSize = origDbSize - nFreeList; + TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origDbSize, finDbSize)); + + /* Note: This is temporary code for use during development of auto-vacuum. + ** + ** Inspect the pointer map to make sure there are no root pages with a + ** page number greater than finDbSize. If so, the auto-vacuum cannot + ** proceed. This limitation will be fixed when root pages are automatically + ** allocated at the start of the database file. + */ + for( i=finDbSize+1; i<=origDbSize; i++ ){ + rc = ptrmapGet(pBt, i, &eType, 0); + if( rc!=SQLITE_OK ) goto autovacuum_out; + if( eType==PTRMAP_ROOTPAGE ){ + TRACE(("AUTOVACUUM: Cannot proceed due to root-page on page %d\n", i)); + return SQLITE_OK; + } + } + + /* Variable 'finDbSize' will be the size of the file in pages after + ** the auto-vacuum has completed (the current file size minus the number + ** of pages on the free list). Loop through the pages that lie beyond + ** this mark, and if they are not already on the free list, move them + ** to a free page earlier in the file (somewhere before finDbSize). + */ + for( iDbPage=finDbSize+1; iDbPage<=origDbSize; iDbPage++ ){ + rc = ptrmapGet(pBt, iDbPage, &eType, &iPtrPage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + assert( eType!=PTRMAP_ROOTPAGE ); + + /* If iDbPage is already on the free-list, do not swap it. */ + if( eType==PTRMAP_FREEPAGE ){ + continue; + } + rc = getPage(pBt, iDbPage, &pDbMemPage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + pDbPage = pDbMemPage->aData; + + /* Find the next page in the free-list that is not already at the end + ** of the file. A page can be pulled off the free list using the + ** allocatePage() routine. + */ + do{ + if( pFreeMemPage ){ + releasePage(pFreeMemPage); + pFreeMemPage = 0; + } + rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0); + if( rc!=SQLITE_OK ) goto autovacuum_out; + assert( iFreePage<=origDbSize ); + }while( iFreePage>finDbSize ); + + /* Move page iDbPage from it's current location to page number iFreePage */ + TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d)\n", + iDbPage, iFreePage, iPtrPage)); + releasePage(pFreeMemPage); + pFreeMemPage = 0; + rc = sqlite3pager_movepage(pPager, pDbPage, iFreePage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + pDbMemPage->pgno = iFreePage; + + /* If pDbPage was a btree-page, then it may have child pages and/or cells + ** that point to overflow pages. The pointer map entries for all these + ** pages need to be changed. + */ + if( eType==PTRMAP_BTREE ){ + rc = setChildPtrmaps(pDbMemPage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + } + releasePage(pDbMemPage); + pDbMemPage = 0; + + /* Fix the database pointer on page iPtrPage that pointed at iDbPage so + ** that it points at iFreePage. Also fix the pointer map entry for + ** iPtrPage. + */ + rc = getPage(pBt, iPtrPage, &pPtrMemPage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + rc = sqlite3pager_write(pPtrMemPage->aData); + if( rc!=SQLITE_OK ) goto autovacuum_out; + modifyPagePointer(pPtrMemPage, iDbPage, iFreePage, eType); + rc = ptrmapPut(pBt, iFreePage, eType, iPtrPage); + if( rc!=SQLITE_OK ) goto autovacuum_out; + releasePage(pPtrMemPage); + } + + /* The entire free-list has been swapped to the end of the file. So + ** truncate the database file to finDbSize pages and consider the + ** free-list empty. + */ + rc = sqlite3pager_write(pBt->pPage1->aData); + if( rc!=SQLITE_OK ) goto autovacuum_out; + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + rc = sqlite3pager_truncate(pBt->pPager, finDbSize); + if( rc!=SQLITE_OK ) goto autovacuum_out; + +autovacuum_out: + /* TODO: A goto autovacuum_out; will fail to call releasePage() on + ** outstanding references. Fix. + */ + if( nRef!=*sqlite3pager_stats(pPager) ){ + sqlite3pager_refdump(pPager); + } + assert( nRef==*sqlite3pager_stats(pPager) ); + if( rc!=SQLITE_OK ){ + sqlite3pager_rollback(pPager); + } + return rc; +} +#endif + /* ** Commit the transaction currently in progress. ** @@ -2472,19 +2763,6 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ return rc; } -/* -** The TRACE macro will print high-level status information about the -** btree operation when the global variable sqlite3_btree_trace is -** enabled. -*/ -#if SQLITE_TEST -# define TRACE(X) if( sqlite3_btree_trace )\ - { sqlite3DebugPrintf X; fflush(stdout); } -#else -# define TRACE(X) -#endif -int sqlite3_btree_trace=0; /* True to enable tracing */ - /* ** Allocate a new page from the database file. ** @@ -2615,6 +2893,15 @@ static int freePage(MemPage *pPage){ n = get4byte(&pPage1->aData[36]); put4byte(&pPage1->aData[36], n+1); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, write an entry in the pointer-map + ** to indicate that the page is free. + */ + if( pBt->autoVacuum ){ + rc = ptrmapPut(pBt, pPage->pgno, PTRMAP_FREEPAGE, 0); + } +#endif + if( n==0 ){ /* This is the first free page */ rc = sqlite3pager_write(pPage->aData); @@ -2757,9 +3044,9 @@ static int fillInCell( */ if( pBt->autoVacuum && rc==0 ){ if( pgnoPtrmap!=0 ){ - rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW, pgnoPtrmap); + rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW2, pgnoPtrmap); }else{ - rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_BTREE, pPage->pgno); + rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno); } } #endif @@ -2867,7 +3154,7 @@ static int reparentChildPages(MemPage *pPage){ parseCellPtr(pPage, pCell, &info); if( info.iOverflow ){ Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); - rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_BTREE, pPage->pgno); + rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno); if( rc!=SQLITE_OK ) return rc; } } @@ -3649,7 +3936,7 @@ static int balance(MemPage *pPage){ if( pPage->nOverflow>0 ){ rc = balance_deeper(pPage); } - if( pPage->nCell==0 ){ + if( rc==SQLITE_OK && pPage->nCell==0 ){ rc = balance_shallower(pPage); } }else{ @@ -3762,7 +4049,9 @@ int sqlite3BtreeInsert( rc = balance(pPage); /* sqlite3BtreePageDump(pCur->pBt, pCur->pgnoRoot, 1); */ /* fflush(stdout); */ - moveToRoot(pCur); + if( rc==SQLITE_OK ){ + moveToRoot(pCur); + } end_insert: sqliteFree(newCell); return rc; @@ -3878,6 +4167,18 @@ int sqlite3BtreeCreateTable(Btree *pBt, int *piTable, int flags){ } rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1); if( rc ) return rc; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* Note: This is temporary code for use during development of auto-vacuum. + ** There should be no need for a pointer map entry for root-pages. + */ + if( pBt->autoVacuum ){ + rc = ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0); + if( rc ){ + sqlite3pager_unref(pRoot->aData); + return rc; + } + } +#endif assert( sqlite3pager_iswriteable(pRoot->aData) ); zeroPage(pRoot, flags | PTF_LEAF); sqlite3pager_unref(pRoot->aData); @@ -4342,25 +4643,38 @@ static void checkList( } if( isFreeList ){ int n = get4byte(&pOvfl[4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif if( n>pCheck->pBt->usableSize/4-8 ){ checkAppendMsg(pCheck, zContext, "freelist leaf count too big on page %d", iPage); N--; }else{ for(i=0; ipBt->autoVacuum ){ + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + } +#endif + checkRef(pCheck, iFreePage, zContext); } N -= n; } } #ifndef SQLITE_OMIT_AUTOVACUUM - /* If this database supports auto-vacuum and iPage is not the last - ** page in this overflow list, check that the pointer-map entry for - ** the following page matches iPage. - */ - if( pCheck->pBt->autoVacuum && !isFreeList && N>0 ){ - i = get4byte(pOvfl); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW, iPage, zContext); + else{ + /* If this database supports auto-vacuum and iPage is not the last + ** page in this overflow list, check that the pointer-map entry for + ** the following page matches iPage. + */ + if( pCheck->pBt->autoVacuum && N>0 ){ + i = get4byte(pOvfl); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + } } #endif iPage = get4byte(pOvfl); @@ -4448,7 +4762,7 @@ static int checkTreePage( Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); } #endif checkList(pCheck, 0, pgnoOvfl, nPage, zContext); @@ -4475,7 +4789,7 @@ static int checkTreePage( sprintf(zContext, "On page %d at right child: ", iPage); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, 0); } #endif checkTreePage(pCheck, pgno, pPage, zContext,0,0,0,0); @@ -4569,6 +4883,12 @@ char *sqlite3BtreeIntegrityCheck(Btree *pBt, int *aRoot, int nRoot){ */ for(i=0; iautoVacuum && aRoot[i]>1 ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + } +#endif checkTreePage(&sCheck, aRoot[i], 0, "List of tree roots: ", 0,0,0,0); } @@ -4711,6 +5031,12 @@ int sqlite3BtreeIsInStmt(Btree *pBt){ */ int sqlite3BtreeSync(Btree *pBt, const char *zMaster){ if( pBt->inTrans==TRANS_WRITE ){ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + int rc = autoVacuumCommit(pBt); + if( rc!=SQLITE_OK ) return rc; + } +#endif return sqlite3pager_sync(pBt->pPager, zMaster); } return SQLITE_OK; diff --git a/src/pager.c b/src/pager.c index 2395649ade..e6ab35b4a5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.169 2004/10/31 02:22:49 drh Exp $ +** @(#) $Id: pager.c,v 1.170 2004/11/02 12:56:41 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -2017,10 +2017,22 @@ static int pager_write_pagelist(PgHdr *pList){ while( pList ){ assert( pList->dirty ); sqlite3OsSeek(&pPager->fd, (pList->pgno-1)*(i64)pPager->pageSize); - CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6); - TRACE3("STORE %d page %d\n", pPager->fd.h, pList->pgno); - rc = sqlite3OsWrite(&pPager->fd, PGHDR_TO_DATA(pList), pPager->pageSize); - CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 0); + /* If there are dirty pages in the page cache with page numbers greater + ** than Pager.dbSize, this means sqlite3pager_truncate() was called to + ** make the file smaller (presumably by auto-vacuum code). Do not write + ** any such pages to the file. + */ + if( pList->pgno<=pPager->dbSize ){ + CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6); + TRACE3("STORE %d page %d\n", pPager->fd.h, pList->pgno); + rc = sqlite3OsWrite(&pPager->fd, PGHDR_TO_DATA(pList), pPager->pageSize); + CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 0); + } +#ifndef NDEBUG + else{ + TRACE3("NOSTORE %d page %d\n", pPager->fd.h, pList->pgno); + } +#endif if( rc ) return rc; pList->dirty = 0; pList = pList->pDirty; @@ -3202,6 +3214,81 @@ sync_exit: return rc; } +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Move the page identified by pData to location pgno in the file. +** +** There must be no references to the current page pgno. If current page +** pgno is not already in the rollback journal, it is not written there by +** by this routine. The same applies to the page pData refers to on entry to +** this routine. +** +** References to the page refered to by pData remain valid. Updating any +** meta-data associated with page pData (i.e. data stored in the nExtra bytes +** allocated along with the page) is the responsibility of the caller. +** +** A transaction must be active when this routine is called, however it is +** illegal to call this routine if a statment transaction is active. +*/ +int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){ + PgHdr *pPg = DATA_TO_PGHDR(pData); + PgHdr *pPgOld; + + assert( !pPager->stmtInUse ); + /* assert( pPg->pNextFree==0 && pPg->pPrevFree==0 && pPg->nRef>0 ); */ + assert( pPg->nRef>0 ); + + /* Unlink pPg from it's hash-chain */ + if( pPg->pNextHash ){ + pPg->pNextHash->pPrevHash = pPg->pPrevHash; + } + if( pPg->pPrevHash ){ + pPg->pPrevHash->pNextHash = pPg->pNextHash; + }else{ + int h = pager_hash(pPg->pgno); + assert( pPager->aHash[h]==pPg ); + pPager->aHash[h] = pPg->pNextHash; + } + + /* Change the page number for pPg */ + pPg->pgno = pgno; + + pPgOld = pager_lookup(pPager, pgno); + if( pPgOld ){ + /* Remove pPgOld from the page number hash-chain and insert pPg. */ + assert(pPgOld->nRef==0 && !pPgOld->pNextStmt && !pPgOld->pPrevStmt ); + if( pPgOld->pNextHash ){ + pPgOld->pNextHash->pPrevHash = pPg; + } + if( pPgOld->pPrevHash ){ + pPgOld->pPrevHash->pNextHash = pPg; + }else{ + int h = pager_hash(pgno); + assert( pPager->aHash[h]==pPgOld ); + pPager->aHash[h] = pPg; + } + pPgOld->pNextHash = pPgOld->pPrevHash = 0; + }else{ + /* Insert pPg into it's new hash-chain. */ + int h = pager_hash(pgno); + if( pPager->aHash[h] ){ + pPager->aHash[h]->pNextHash = pPg; + } + pPg->pNextHash = pPager->aHash[h]; + pPg->pPrevHash = 0; + } + + /* Don't write the old page when sqlite3pager_sync() is called. Do write + ** the new one. + */ + pPgOld->dirty = 0; + pPg->dirty = 1; + pPager->dirtyCache = 1; + + return SQLITE_OK; +} +#endif + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Return the current state of the file lock for the given pager. diff --git a/src/pager.h b/src/pager.h index ff6dd87338..23c0dc6b56 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.38 2004/10/05 02:41:43 drh Exp $ +** @(#) $Id: pager.h,v 1.39 2004/11/02 12:56:41 danielk1977 Exp $ */ /* @@ -91,6 +91,7 @@ const char *sqlite3pager_dirname(Pager*); const char *sqlite3pager_journalname(Pager*); int sqlite3pager_rename(Pager*, const char *zNewName); void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*); +int sqlite3pager_movepage(Pager*,void*,Pgno); #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) int sqlite3pager_lockstate(Pager*); diff --git a/test/autovacuum.test b/test/autovacuum.test new file mode 100644 index 0000000000..85a18c7c18 --- /dev/null +++ b/test/autovacuum.test @@ -0,0 +1,99 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the SELECT statement. +# +# $Id: autovacuum.test,v 1.1 2004/11/02 12:56:41 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc make_str {char len} { + set str [string repeat $char. $len] + return [string range $str 0 [expr $len-1]] +} + +proc file_pages {} { + return [expr [file size test.db] / 1024] +} + +do_test autovacuum-1.1 { + execsql { + CREATE TABLE av1(a); + } +} {} + +set ENTRY_LEN 1100 + +set delete_orders [list] +lappend delete_orders {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20} +lappend delete_orders {20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} +lappend delete_orders {8 18 2 4 14 11 13 3 10 7 9 5 12 17 19 15 20 6 16 1} +lappend delete_orders {10 3 11 17 19 20 7 4 13 6 1 14 16 12 9 18 8 15 5 2} + +lappend delete_orders {{1 2 3 4 5 6 7 8 9 10} {11 12 13 14 15 16 17 18 19 20}} +lappend delete_orders \ + {{19 8 17 15} {16 11 9 14} {18 5 3 1} {13 20 7 2} {6 12 4 10}} + + +set tn 0 +foreach delete_order $delete_orders { + incr tn + + # Set up the table. + set ::tbl_data [list] + foreach i [lsort -integer [eval concat $delete_order]] { + execsql "INSERT INTO av1 (oid, a) VALUES($i, '[make_str $i $ENTRY_LEN]')" + lappend ::tbl_data [make_str $i $ENTRY_LEN] + } + +# puts "File has [file_pages] pages" + + do_test autovacuum-1.$tn.1 { + execsql { + pragma integrity_check + } + } {ok} + + foreach delete $delete_order { +# if {$delete==6} { set btree_trace 1 ; breakpoint } + do_test autovacuum-1.$tn.($delete).1 { + execsql " + DELETE FROM av1 WHERE oid IN ([join $delete ,]) + " + } {} +set btree_trace 0 + + do_test autovacuum-1.$tn.($delete).2 { + execsql { + pragma integrity_check + } + } {ok} + + foreach d $delete { + set idx [lsearch $::tbl_data [make_str $d $ENTRY_LEN]] + set ::tbl_data [lreplace $::tbl_data $idx $idx] + } + do_test autovacuum-1.$tn.($delete).3 { + execsql { + select a from av1 + } + } $::tbl_data +# if {$::nErr>0} finish_test + } + + do_test autovacuum-1.$tn.3 { + file_pages + } {3} +} + +finish_test + diff --git a/test/quick.test b/test/quick.test index 7fd641d17d..a8514dac9b 100644 --- a/test/quick.test +++ b/test/quick.test @@ -10,7 +10,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: quick.test,v 1.30 2004/09/02 14:57:09 drh Exp $ +# $Id: quick.test,v 1.31 2004/11/02 12:56:41 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -32,6 +32,8 @@ set EXCLUDE { misuse.test quick.test utf16.test + + autovacuum.test } if {[sqlite3 -has-codec]} {