diff --git a/manifest b/manifest index 628b5d06c7..a9a6a918bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Don't\scode\san\sOP_Statement\swithin\ssqlite3NestedParse().\sAlso\sa\scorrection\nto\sthe\sUPDATE\sstatement\sused\swithin\sdestroyRootPage().\s(CVS\s2064) -D 2004-11-05T09:19:28 +C Fix\sallocation\sof\stables\sin\san\sauto-vacuum\sdatabase\swhen\sthe\srequired\sroot-page\sis\son\sthe\sfree-list.\s(CVS\s2065) +D 2004-11-05T12:27:02 F Makefile.in c4d2416860f472a1e3393714d0372074197565df F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1 @@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea -F src/btree.c f97b5a3919147fe36f776d08c80212ba3ea883aa +F src/btree.c 659bb0c16b7b2429ff93a2260a051576ccde0a0b F src/btree.h 3166388fa58c5594d8064d38b43440d79da38fb6 F src/build.c dc8b9ab836f2323d9b313c2d703b00b2e9441382 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad @@ -87,7 +87,7 @@ F test/attach.test e305dd59a375e37c658c6d401f19f8a95880bf9a F test/attach2.test 399128a7b3b209a339a8dbf53ca2ed42eb982d1a F test/attach3.test 287af46653e7435b2d1eda10d8115dcc8a6883e2 F test/auth.test 1cc252d9e7b3bdc1314199cbf3a0d3c5ed026c21 -F test/autovacuum.test a5b11269daac313bea6694b04473fdd0e16e439a +F test/autovacuum.test b2ba86ec6ab2734232f299769be0c7c0c41939a1 F test/bigfile.test d3744a8821ce9abb8697f2826a3e3d22b719e89f F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 F test/bind.test fa74f98417cd313f28272acff832a8a7d04a0916 @@ -252,7 +252,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c -P 296a298c484aac981e7e490a0cb4159717fc4ea4 -R cb8380605519101724d006bec5a8cc07 +P fdcc31f0c6106dacfed6612b173fe4be3c02546a +R 07fa558cc5210292e3d2899ca22b721c U danielk1977 -Z 36305ac9fc261446032e2140c068e8dd +Z e28ed5c0b538966881c79d2d34296784 diff --git a/manifest.uuid b/manifest.uuid index 1b683be49a..897aacf19e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdcc31f0c6106dacfed6612b173fe4be3c02546a \ No newline at end of file +4e2433378e06210f0274c317c6d12b48236211fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ba2fe9ee67..5d0e37971a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.207 2004/11/05 03:56:01 drh Exp $ +** $Id: btree.c,v 1.208 2004/11/05 12:27:02 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -1682,7 +1682,7 @@ static int relocatePage( } /* Forward declaration required by autoVacuumCommit(). */ -static int allocatePage(Btree *, MemPage **, Pgno *, Pgno); +static int allocatePage(Btree *, MemPage **, Pgno *, Pgno, u8); /* ** This routine is called prior to sqlite3pager_commit when a transaction @@ -1694,7 +1694,6 @@ static int autoVacuumCommit(Btree *pBt){ int nPtrMap; /* Number of pointer-map pages deallocated */ Pgno origSize; /* Pages in the database file */ Pgno finSize; /* Pages in the database file after truncation */ - int i; /* Counter variable */ int rc; /* Return code */ u8 eType; int pgsz = pBt->pageSize; /* Page size for this database */ @@ -1722,27 +1721,8 @@ static int autoVacuumCommit(Btree *pBt){ origSize = sqlite3pager_pagecount(pPager); nPtrMap = (nFreeList-origSize+PTRMAP_PAGENO(pgsz, origSize)+pgsz/5)/(pgsz/5); finSize = origSize - nFreeList - nPtrMap; - TRACE(("AUTOVACUUM: Begin (db size %d->%d)\n", origSize, finSize)); -#if 0 - /* Note: This is temporary code for use during development of auto-vacuum. - ** - ** Inspect the pointer map to make sure there are no root pages with a - ** page number greater than finSize. If so, the auto-vacuum cannot - ** proceed. This limitation will be fixed when root pages are automatically - ** allocated at the start of the database file. - */ - for( i=finSize+1; i<=origSize; i++ ){ - rc = ptrmapGet(pBt, i, &eType, 0); - if( rc!=SQLITE_OK ) goto autovacuum_out; - if( eType==PTRMAP_ROOTPAGE ){ - TRACE(("AUTOVACUUM: Cannot proceed due to root-page on page %d\n", i)); - return SQLITE_OK; - } - } -#endif - /* Variable 'finSize' will be the size of the file in pages after ** the auto-vacuum has completed (the current file size minus the number ** of pages on the free list). Loop through the pages that lie beyond @@ -1755,7 +1735,7 @@ static int autoVacuumCommit(Btree *pBt){ assert( eType!=PTRMAP_ROOTPAGE ); /* If iDbPage is a free or pointer map page, do not swap it. - ** Instead, make sure the page is in the journal file. + ** TODO: Instead, make sure the page is in the journal file. */ if( eType==PTRMAP_FREEPAGE || PTRMAP_ISPAGE(pgsz, iDbPage) ){ continue; @@ -1772,7 +1752,7 @@ static int autoVacuumCommit(Btree *pBt){ releasePage(pFreeMemPage); pFreeMemPage = 0; } - rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0); + rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0, 0); if( rc!=SQLITE_OK ) goto autovacuum_out; assert( iFreePage<=origSize ); }while( iFreePage>finSize ); @@ -2842,8 +2822,18 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. +** +** If the "exact" parameter is not 0, and the page-number nearby exists +** anywhere on the free-list, then it is guarenteed to be returned. This +** is only used by auto-vacuum databases when allocating a new table. */ -static int allocatePage(Btree *pBt, MemPage **ppPage, Pgno *pPgno, Pgno nearby){ +static int allocatePage( + Btree *pBt, + MemPage **ppPage, + Pgno *pPgno, + Pgno nearby, + u8 exact +){ MemPage *pPage1; int rc; int n; /* Number of pages on the freelist */ @@ -2853,63 +2843,169 @@ static int allocatePage(Btree *pBt, MemPage **ppPage, Pgno *pPgno, Pgno nearby){ n = get4byte(&pPage1->aData[36]); if( n>0 ){ /* There are pages on the freelist. Reuse one of those pages. */ - MemPage *pTrunk; + MemPage *pTrunk = 0; + Pgno iTrunk; + MemPage *pPrevTrunk = 0; + u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + + /* If the 'exact' parameter was true and a query of the pointer-map + ** shows that the page 'nearby' is somewhere on the free-list, then + ** the entire-list will be searched for that page. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( exact ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + *pPgno = nearby; + } +#endif + + /* Decrement the free-list count by 1. Set iTrunk to the index of the + ** first free-list trunk page. iPrevTrunk is initially 1. + */ rc = sqlite3pager_write(pPage1->aData); if( rc ) return rc; put4byte(&pPage1->aData[36], n-1); - rc = getPage(pBt, get4byte(&pPage1->aData[32]), &pTrunk); - if( rc ) return rc; - rc = sqlite3pager_write(pTrunk->aData); - if( rc ){ - releasePage(pTrunk); - return rc; - } - k = get4byte(&pTrunk->aData[4]); - if( k==0 ){ - /* The trunk has no leaves. So extract the trunk page itself and - ** use it as the newly allocated page */ - *pPgno = get4byte(&pPage1->aData[32]); - memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); - *ppPage = pTrunk; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); - }else if( k>pBt->usableSize/4 - 8 ){ - /* Value of k is out of range. Database corruption */ - return SQLITE_CORRUPT; /* bkpt-CORRUPT */ - }else{ - /* Extract a leaf from the trunk */ - int closest; - unsigned char *aData = pTrunk->aData; - if( nearby>0 ){ - int i, dist; - closest = 0; - dist = get4byte(&aData[8]) - nearby; - if( dist<0 ) dist = -dist; - for(i=1; iaData[0]); }else{ - closest = 0; + iTrunk = get4byte(&pPage1->aData[32]); } - *pPgno = get4byte(&aData[8+closest*4]); - if( *pPgno>sqlite3pager_pagecount(pBt->pPager) ){ - /* Free page off the end of the file */ + rc = getPage(pBt, iTrunk, &pTrunk); + if( rc ){ + releasePage(pPrevTrunk); + return rc; + } + + /* TODO: This should move to after the loop? */ + rc = sqlite3pager_write(pTrunk->aData); + if( rc ){ + releasePage(pTrunk); + releasePage(pPrevTrunk); + return rc; + } + + k = get4byte(&pTrunk->aData[4]); + if( k==0 && !searchList ){ + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly + ** allocated page */ + assert( pPrevTrunk==0 ); + *pPgno = iTrunk; + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + *ppPage = pTrunk; + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + }else if( k>pBt->usableSize/4 - 8 ){ + /* Value of k is out of range. Database corruption */ return SQLITE_CORRUPT; /* bkpt-CORRUPT */ +#ifndef SQLITE_OMIT_AUTOVACUUM + }else if( searchList && nearby==iTrunk ){ + /* The list is being searched and this trunk page is the page + ** to allocate, regardless of whether it has leaves. + */ + assert( *pPgno==iTrunk ); + *ppPage = pTrunk; + searchList = 0; + if( k==0 ){ + if( !pPrevTrunk ){ + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + }else{ + memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); + } + }else{ + /* The trunk page is required by the caller but it contains + ** pointers to free-list leaves. The first leaf becomes a trunk + ** page in this case. + */ + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + rc = getPage(pBt, iNewTrunk, &pNewTrunk); + if( rc!=SQLITE_OK ){ + releasePage(pTrunk); + releasePage(pPrevTrunk); + return rc; + } + rc = sqlite3pager_write(pNewTrunk->aData); + if( rc!=SQLITE_OK ){ + releasePage(pNewTrunk); + releasePage(pTrunk); + releasePage(pPrevTrunk); + return rc; + } + memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); + put4byte(&pNewTrunk->aData[4], k-1); + memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); + if( !pPrevTrunk ){ + put4byte(&pPage1->aData[32], iNewTrunk); + }else{ + put4byte(&pPrevTrunk->aData[0], iNewTrunk); + } + releasePage(pNewTrunk); + } + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); +#endif + }else{ + /* Extract a leaf from the trunk */ + int closest; + Pgno iPage; + unsigned char *aData = pTrunk->aData; + if( nearby>0 ){ + int i, dist; + closest = 0; + dist = get4byte(&aData[8]) - nearby; + if( dist<0 ) dist = -dist; + for(i=1; isqlite3pager_pagecount(pBt->pPager) ){ + /* Free page off the end of the file */ + return SQLITE_CORRUPT; /* bkpt-CORRUPT */ + } + TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" + ": %d more free pages\n", + *pPgno, closest+1, k, pTrunk->pgno, n-1)); + if( closestaData); + rc = sqlite3pager_write((*ppPage)->aData); + } + searchList = 0; + } } - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d: %d more free pages\n", - *pPgno, closest+1, k, pTrunk->pgno, n-1)); - if( closestaData); - rc = sqlite3pager_write((*ppPage)->aData); - } - } + releasePage(pPrevTrunk); + }while( searchList ); + releasePage(pTrunk); }else{ /* There are no pages on the freelist, so create a new page at the ** end of the file */ @@ -2958,14 +3054,11 @@ static int freePage(MemPage *pPage){ #ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, write an entry in the pointer-map - ** to indicate that the page is free. Also make sure the page is in - ** the journal file. + ** to indicate that the page is free. */ if( pBt->autoVacuum ){ rc = ptrmapPut(pBt, pPage->pgno, PTRMAP_FREEPAGE, 0); if( rc ) return rc; - rc = sqlite3pager_write(pPage->aData); - if( rc ) return rc; } #endif @@ -3102,7 +3195,7 @@ static int fillInCell( #ifndef SQLITE_OMIT_AUTOVACUUM Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ #endif - rc = allocatePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl); + rc = allocatePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); #ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map @@ -3710,7 +3803,7 @@ static int balance_nonroot(MemPage *pPage){ apOld[i] = 0; sqlite3pager_write(pNew->aData); }else{ - rc = allocatePage(pBt, &pNew, &pgnoNew[i], pgnoNew[i-1]); + rc = allocatePage(pBt, &pNew, &pgnoNew[i], pgnoNew[i-1], 0); if( rc ) goto balance_cleanup; apNew[i] = pNew; } @@ -3972,7 +4065,7 @@ static int balance_deeper(MemPage *pPage){ assert( pPage->pParent==0 ); assert( pPage->nOverflow>0 ); pBt = pPage->pBt; - rc = allocatePage(pBt, &pChild, &pgnoChild, pPage->pgno); + rc = allocatePage(pBt, &pChild, &pgnoChild, pPage->pgno, 0); if( rc ) return rc; assert( sqlite3pager_iswriteable(pChild->aData) ); usableSize = pBt->usableSize; @@ -4241,26 +4334,13 @@ int sqlite3BtreeCreateTable(Btree *pBt, int *piTable, int flags){ return SQLITE_READONLY; } #ifdef SQLITE_OMIT_AUTOVACUUM - rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1); + rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1, 0); if( rc ) return rc; #else if( pBt->autoVacuum ){ Pgno pgnoMove; /* Move a page here to make room for the root-page */ MemPage *pPageMove; /* The page to move to. */ - /* Run the auto-vacuum code to ensure the free-list is empty. This is - ** not really necessary, but it avoids complications in dealing with - ** a free-list in the code below. - ** TODO: This may need to be revisited. - ** TODO2: Actually this is no-good. running the auto-vacuum routine - ** involves truncating the database, which means the journal-file - ** must be synced(). No-good. - */ -/* - rc = autoVacuumCommit(pBt); - if( rc!=SQLITE_OK ) return rc; -*/ - /* Read the value of meta[3] from the database to determine where the ** root page of the new table should go. meta[3] is the largest root-page ** created so far, so the new root-page is (meta[3]+1). @@ -4279,7 +4359,7 @@ int sqlite3BtreeCreateTable(Btree *pBt, int *piTable, int flags){ ** be moved to the allocated page (unless the allocated page happens ** to reside at pgnoRoot). */ - rc = allocatePage(pBt, &pPageMove, &pgnoMove, 1); + rc = allocatePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1); if( rc!=SQLITE_OK ){ return rc; } @@ -4329,7 +4409,7 @@ int sqlite3BtreeCreateTable(Btree *pBt, int *piTable, int flags){ return rc; } }else{ - rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1); + rc = allocatePage(pBt, &pRoot, &pgnoRoot, 1, 0); if( rc ) return rc; } #endif diff --git a/test/autovacuum.test b/test/autovacuum.test index 6b80173b5d..1fe7c0bc85 100644 --- a/test/autovacuum.test +++ b/test/autovacuum.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the SELECT statement. # -# $Id: autovacuum.test,v 1.6 2004/11/04 14:30:06 danielk1977 Exp $ +# $Id: autovacuum.test,v 1.7 2004/11/05 12:27:03 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -79,6 +79,7 @@ foreach delete_order $delete_orders { } } {ok} +# set btree_trace 1 foreach delete $delete_order { # Delete one set of rows from the table. do_test autovacuum-1.$tn.($delete).1 { @@ -112,37 +113,141 @@ foreach delete_order $delete_orders { } {4} } -# Tests autovacuum-2.* test that root pages are allocated correctly at -# the start of the file. -do_test autovacuum-2.1 { - for {set i 0} {$i<5} {incr i} { - execsql " - INSERT INTO av1 VALUES('[make_str abc 1000]') - " +# Tests cases autovacuum-2.* test that root pages are allocated +# and deallocated correctly at the start of the file. Operation is roughly as +# follows: +# +# autovacuum-2.1.*: Drop the tables that currently exist in the database. +# autovacuum-2.2.*: Create some tables. Ensure that data pages can be +# moved correctly to make space for new root-pages. +# autovacuum-2.3.*: Drop one of the tables just created (not the last one), +# and check that one of the other tables is moved to +# the free root-page location. +# autovacuum-2.4.*: Check that a table can be created correctly when the +# root-page it requires is on the free-list. +# +do_test autovacuum-2.1.1 { + execsql { + DROP TABLE av1; } - file_pages -} {14} +} {} +do_test autovacuum-2.1.2 { + file_pages +} {1} -for {set i 5} {$i < 15} {incr i} { - set tablename "av$i" - - do_test autovacuum-2.$i.2 { - execsql " - CREATE TABLE $tablename (a); - SELECT rootpage FROM sqlite_master WHERE name = '$tablename'; - " - } $i +# Create a table and put some data in it. +do_test autovacuum-2.2.1 { + execsql { + CREATE TABLE av1(x); + SELECT rootpage FROM sqlite_master ORDER BY rootpage; + } +} {3} +do_test autovacuum-2.2.2 { + execsql " + INSERT INTO av1 VALUES('[make_str abc 3000]'); + INSERT INTO av1 VALUES('[make_str def 3000]'); + INSERT INTO av1 VALUES('[make_str ghi 3000]'); + INSERT INTO av1 VALUES('[make_str jkl 3000]'); + " + set ::av1_data [db eval {select * from av1}] + file_pages +} {15} - do_test autovacuum-2.$i.3 { - file_pages - } [expr $i+10] +# Create another table. Check it is located immediately after the first. +# This test case moves the second page in an over-flow chain. +do_test autovacuum-2.2.3 { + execsql { + CREATE TABLE av2(x); + SELECT rootpage FROM sqlite_master ORDER BY rootpage; + } +} {3 4} +do_test autovacuum-2.2.4 { + file_pages +} {16} - do_test autovacuum-2.$i.4 { - execsql { - pragma integrity_check - } - } {ok} -} +# Create another table. Check it is located immediately after the second. +# This test case moves the first page in an over-flow chain. +do_test autovacuum-2.2.5 { + execsql { + CREATE TABLE av3(x); + SELECT rootpage FROM sqlite_master ORDER BY rootpage; + } +} {3 4 5} +do_test autovacuum-2.2.6 { + file_pages +} {17} + +# Create another table. Check it is located immediately after the second. +# This test case moves a btree leaf page. +do_test autovacuum-2.2.7 { + execsql { + CREATE TABLE av4(x); + SELECT rootpage FROM sqlite_master ORDER BY rootpage; + } +} {3 4 5 6} +do_test autovacuum-2.2.8 { + file_pages +} {18} +do_test autovacuum-2.2.9 { + execsql { + select * from av1 + } +} $av1_data + +do_test autovacuum-2.3.1 { + execsql { + INSERT INTO av2 SELECT 'av1' || x FROM av1; + INSERT INTO av3 SELECT 'av2' || x FROM av1; + INSERT INTO av4 SELECT 'av3' || x FROM av1; + } + set ::av2_data [execsql {select x from av2}] + set ::av3_data [execsql {select x from av3}] + set ::av4_data [execsql {select x from av4}] + file_pages +} {54} +do_test autovacuum-2.3.2 { + execsql { + DROP TABLE av2; + SELECT rootpage FROM sqlite_master ORDER BY rootpage; + } +} {3 4 5} +do_test autovacuum-2.3.3 { + file_pages +} {41} +do_test autovacuum-2.3.4 { + execsql { + SELECT x FROM av3; + } +} $::av3_data +do_test autovacuum-2.3.5 { + execsql { + SELECT x FROM av4; + } +} $::av4_data + +# Drop all the tables in the file. This puts all pages except the first 2 +# (the sqlite_master root-page and the first pointer map page) on the +# free-list. +do_test autovacuum-2.4.1 { + execsql { + DROP TABLE av1; + DROP TABLE av3; + BEGIN; + DROP TABLE av4; + } + file_pages +} {15} +do_test autovacuum-2.4.2 { + for {set i 3} {$i<=10} {incr i} { + execsql "CREATE TABLE av$i (x)" + } + file_pages +} {15} +do_test autovacuum-2.4.3 { + execsql { + SELECT rootpage FROM sqlite_master ORDER by rootpage + } +} {3 4 5 6 7 8 9 10} finish_test