diff --git a/manifest b/manifest index e27c4059e2..2cf2fc7e78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scoverage\simprovements.\s(CVS\s2215) -D 2005-01-15T01:52:32 +C Enhance\sthe\sperformance\sof\sauto-vacuum\sdatabases\sby\sreducing\sthe\snumber\sof\spointer-map\sentries\swritten\sduring\stree\sbalancing.\sAlso\sfix\sbugs\sin\sbalance_quick().\s(CVS\s2216) +D 2005-01-15T12:45:51 F Makefile.in 6ce51dde6a8fe82fc12f20dec750572f6a19f56a F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1 @@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea -F src/btree.c 01027e3c1b9f53f17eec964da2a3e45d253b2b87 +F src/btree.c 426032061fb39b1bfcbf1d58915d5a6346bb4e22 F src/btree.h 74d19cf40ab49fd69abe9e4e12a6c321ad86c497 F src/build.c af1296e8a21a406b4f4c4f1e1365e075071219f3 F src/cursor.c f883813759742068890b1f699335872bfa8fdf41 @@ -91,10 +91,10 @@ F test/attach2.test eeb987770f4dbe68bd29afdbc2e8cff0142e6eb5 F test/attach3.test c05c70b933afbde0901dab9da3e66ee842c09f38 F test/auth.test 559e0816b8100740624ebb0ab7aab05f5c92831c F test/autoinc.test c071e51ff167b8e889212273588d9cca71845b70 -F test/autovacuum.test a4e8da39a6268378c4f9fc17fe2df1d5be16d631 +F test/autovacuum.test a15021f685f2b3be5ad120f35b5a9f413a950702 F test/autovacuum_crash.test 2dca85cbcc497098e45e8847c86407eb3554f3d4 F test/autovacuum_ioerr.test 55ea907df34edb9be78a910a1636c2eb3c17ecc4 -F test/autovacuum_ioerr2.test bf427c86e4daa8638a2eb849bbe1446c234c73d3 +F test/autovacuum_ioerr2.test ce5d4ff67a022f02dd594c15bf246e50680d8a65 F test/bigfile.test d3744a8821ce9abb8697f2826a3e3d22b719e89f F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 F test/bind.test 578c6526f9e7298a8993815336d676a12684b0cd @@ -117,7 +117,7 @@ F test/collate6.test 6c9470d1606ee3e564675b229653e320c49ec638 F test/conflict.test c5b849b01cfbe0a4f63a90cba6f68e2fe3a75f87 F test/corrupt.test 0080ddcece23e8ba47c44608c4fb73fd4d1d8ce2 F test/corrupt2.test cb1f813df7559de3021e01170af0bba31507a9a5 -F test/crash.test b87f2c2fe6a05c46c8832bb077e131bb4b507a8d +F test/crash.test 03aa56f873bdf9ca1beda5c5a76a6500106f2844 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/cursor.test d7c65ea0fc4e321e12fbcf5c7f3e2211ef45379b F test/date.test ef6c679d0b59502457dbd78ee1c3c085c949c4c4 @@ -269,7 +269,7 @@ F www/tclsqlite.tcl e73f8f8e5f20e8277619433f7970060ab01088fc F www/vdbe.tcl 095f106d93875c94b47367384ebc870517431618 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/whentouse.tcl c3b50d3ac31c54be2a1af9b488a89d22f1e6e746 -P 3ef95d5fe98e7451f8b87b3f5259163f3e7d0289 -R 0dac24ab9c449997916f810d80271d4c -U drh -Z 1f96a701f8d82044d6c8d59082332ad8 +P 92f9d2b2f480fccfa6e8b70a1d19058b92a4ea8f +R 0a0af1f143c2a17b3aa82ad2aba89b86 +U danielk1977 +Z 0cdc37c80cb8604b1387cf29891ba583 diff --git a/manifest.uuid b/manifest.uuid index 3e736c8c5b..fd6c06a026 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92f9d2b2f480fccfa6e8b70a1d19058b92a4ea8f \ No newline at end of file +0ae29538ccccfc237904cbcfb4507074db0f5905 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b80189304a..d21324e564 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.233 2005/01/14 22:55:49 drh Exp $ +** $Id: btree.c,v 1.234 2005/01/15 12:45:51 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -463,6 +463,7 @@ static int ptrmapPut(Btree *pBt, Pgno key, u8 eType, Pgno pgno){ int offset; /* Offset in pointer map page */ int rc; + assert( pBt->autoVacuum ); assert( key!=0 ); iPtrmap = PTRMAP_PAGENO(pBt->usableSize, key); rc = sqlite3pager_get(pBt->pPager, iPtrmap, (void **)&pPtrmap); @@ -642,6 +643,19 @@ static int cellSizePtr(MemPage *pPage, u8 *pCell){ return info.nSize; } +/* +** If pCell, part of pPage, contains a pointer to an overflow page, +** return the overflow page number. Otherwise return 0. +*/ +static Pgno ovflPagePtr(MemPage *pPage, u8 *pCell){ + CellInfo info; + parseCellPtr(pPage, pCell, &info); + if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){ + return get4byte(&pCell[info.iOverflow]); + } + return 0; +} + /* ** Do sanity checking on a page. Throw an exception if anything is ** not right. @@ -1812,7 +1826,10 @@ static int autoVacuumCommit(Btree *pBt, Pgno *nTrunc){ pFreeMemPage = 0; } rc = allocatePage(pBt, &pFreeMemPage, &iFreePage, 0, 0); - if( rc!=SQLITE_OK ) goto autovacuum_out; + if( rc!=SQLITE_OK ){ + releasePage(pDbMemPage); + goto autovacuum_out; + } assert( iFreePage<=origSize ); }while( iFreePage>finSize ); releasePage(pFreeMemPage); @@ -3320,6 +3337,8 @@ static int reparentPage(Btree *pBt, Pgno pgno, MemPage *pNewParent, int idx){ return SQLITE_OK; } + + /* ** Change the pParent pointer of all children of pPage to point back ** to pPage. @@ -3335,11 +3354,7 @@ static int reparentChildPages(MemPage *pPage){ Btree *pBt = pPage->pBt; int rc = SQLITE_OK; -#ifdef SQLITE_OMIT_AUTOVACUUM if( pPage->leaf ) return SQLITE_OK; -#else - if( !pBt->autoVacuum && pPage->leaf ) return SQLITE_OK; -#endif for(i=0; inCell; i++){ u8 *pCell = findCell(pPage, i); @@ -3347,23 +3362,6 @@ static int reparentChildPages(MemPage *pPage){ rc = reparentPage(pBt, get4byte(pCell), pPage, i); if( rc!=SQLITE_OK ) return rc; } -#ifndef SQLITE_OMIT_AUTOVACUUM - /* If the database supports auto-vacuum, then check each cell to see - ** if it contains a pointer to an overflow page. If so, then the - ** pointer-map must be updated accordingly. - ** - ** TODO: This looks like quite an expensive thing to do. Investigate. - */ - if( pBt->autoVacuum ){ - CellInfo info; - parseCellPtr(pPage, pCell, &info); - if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){ - Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); - rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno); - if( rc!=SQLITE_OK ) return rc; - } - } -#endif } if( !pPage->leaf ){ rc = reparentPage(pBt, get4byte(&pPage->aData[pPage->hdrOffset+8]), @@ -3582,6 +3580,7 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ u8 *pCell; int szCell; CellInfo info; + Btree *pBt = pPage->pBt; u8 parentCell[64]; /* How big should this be? */ int parentIdx = pParent->nCell; @@ -3590,7 +3589,7 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ /* Allocate a new page. Insert the overflow cell from pPage ** into it. Then remove the overflow cell from pPage. */ - rc = allocatePage(pPage->pBt, &pNew, &pgnoNew, 0, 0); + rc = allocatePage(pBt, &pNew, &pgnoNew, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3605,6 +3604,7 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ ** pPage is the next-to-right child. Then balance() the parent ** page, in case it is now overfull. */ + assert( pPage->nCell>0 ); parseCellPtr(pPage, findCell(pPage, pPage->nCell-1), &info); rc = fillInCell(pParent, parentCell, 0, info.nKey, 0, 0, &parentSize); if( rc!=SQLITE_OK ){ @@ -3621,10 +3621,39 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ pNew->pParent = pParent; sqlite3pager_ref(pParent->aData); + if( pBt->autoVacuum ){ + rc = ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno); + if( rc!=SQLITE_OK ){ + return rc; + } + pCell = findCell(pNew, 0); + parseCellPtr(pNew, pCell, &info); + if( info.nData>info.nLocal ){ + Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); + rc = ptrmapPut(pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pgnoNew); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + releasePage(pNew); return balance(pParent, 0); } +/* +** The ISAUTOVACUUM macro is used within balance_nonroot() to determine +** if the database supports auto-vacuum or not. Because it is used +** within an expression that is an argument to another macro +** (sqliteMallocRaw), it is not possible to use conditional compilation. +** So, this macro is defined instead. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM (pBt->autoVacuum) +#else +#define ISAUTOVACUUM 0 +#endif + /* ** This routine redistributes Cells on pPage and up to NN*2 siblings ** of pPage so that all pages have about the same amount of free space. @@ -3685,6 +3714,9 @@ static int balance_nonroot(MemPage *pPage){ int *szCell; /* Local size of all cells in apCell[] */ u8 *aCopy[NB]; /* Space for holding data of apCopy[] */ u8 *aSpace; /* Space to hold copies of dividers cells */ +#ifndef SQLITE_OMIT_VACUUM + u8 *aFrom = 0; +#endif /* ** Find the parent page. @@ -3711,8 +3743,13 @@ static int balance_nonroot(MemPage *pPage){ pPage->leafData && pPage->nOverflow==1 && pPage->aOvfl[0].idx==pPage->nCell && + pPage->pParent->pgno!=1 && get4byte(&pParent->aData[pParent->hdrOffset+8])==pPage->pgno ){ + /* + ** TODO: Check the siblings to the left of pPage. It may be that + ** they are not full and no new page is required. + */ return balance_quick(pPage, pParent); } #endif @@ -3725,6 +3762,7 @@ static int balance_nonroot(MemPage *pPage){ (mxCellPerPage+2)*NB*(sizeof(u8*)+sizeof(int)) + sizeof(MemPage)*NB + pBt->psAligned*(5+NB) + + (ISAUTOVACUUM ? (mxCellPerPage+2)*NN*2 : 0) ); if( apCell==0 ){ return SQLITE_NOMEM; @@ -3735,6 +3773,11 @@ static int balance_nonroot(MemPage *pPage){ aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)]; } aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)]; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + aFrom = &aSpace[5*pBt->psAligned]; + } +#endif /* ** Find the cell in the parent page whose left child points back @@ -3836,6 +3879,18 @@ static int balance_nonroot(MemPage *pPage){ for(j=0; jautoVacuum ){ + int a; + aFrom[nCell] = i; + for(a=0; anOverflow; a++){ + if( pOld->aOvfl[a].pCell==apCell[nCell] ){ + aFrom[nCell] = 0xFF; + break; + } + } + } +#endif nCell++; } if( ipsAligned*5 ); memcpy(pTemp, apDiv[i], sz); apCell[nCell] = pTemp+leafCorrection; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + aFrom[nCell] = 0xFF; + } +#endif dropCell(pParent, nxDiv, sz); szCell[nCell] -= leafCorrection; assert( get4byte(pTemp)==pgnoOld[i] ); @@ -4014,12 +4074,39 @@ static int balance_nonroot(MemPage *pPage){ */ j = 0; for(i=0; ipgno==pgnoNew[i] ); assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]); - j = cntNew[i]; assert( pNew->nCell>0 ); assert( pNew->nOverflow==0 ); + +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If this is an auto-vacuum database, update the pointer map entries + ** that point to the siblings that were rearranged. These can be: left + ** children of cells, the right-child of the page, or overflow pages + ** pointed to by cells. + */ + if( pBt->autoVacuum ){ + for(k=j; kpgno!=pNew->pgno ){ + Pgno ovfl = ovflPagePtr(pNew, findCell(pNew, k-j)); + if( ovfl ){ + rc = ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pNew->pgno); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } + } + } + } +#endif + + j = cntNew[i]; + + /* If the sibling page assembled above was not the right-most sibling, + ** insert a divider cell into the parent page. + */ if( iaData[8], pCell, 4); pTemp = 0; }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ CellInfo info; j--; parseCellPtr(pNew, apCell[j], &info); @@ -4047,6 +4139,21 @@ static int balance_nonroot(MemPage *pPage){ rc = insertCell(pParent, nxDiv, pCell, sz, pTemp, 4); if( rc!=SQLITE_OK ) goto balance_cleanup; put4byte(findOverflowCell(pParent,nxDiv), pNew->pgno); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If this is an auto-vacuum database, and not a leaf-data tree, + ** then update the pointer map with an entry for the overflow page + ** that the cell just inserted points to (if any). + */ + if( pBt->autoVacuum && !leafData ){ + Pgno ovfl = ovflPagePtr(pParent, findOverflowCell(pParent, nxDiv)); + if( ovfl ){ + rc = ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pParent->pgno); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } + } +#endif j++; nxDiv++; } @@ -4076,7 +4183,7 @@ static int balance_nonroot(MemPage *pPage){ /* ** Balance the parent page. Note that the current page (pPage) might - ** have been added to the freelist is it might no longer be initialized. + ** have been added to the freelist so it might no longer be initialized. ** But the parent page will always be initialized. */ assert( pParent->isInit ); @@ -4114,6 +4221,7 @@ static int balance_shallower(MemPage *pPage){ int mxCellPerPage; /* Maximum number of cells per page */ u8 **apCell; /* All cells from pages being balanced */ int *szCell; /* Local size of all cells */ + int i; assert( pPage->pParent==0 ); assert( pPage->nCell==0 ); @@ -4178,6 +4286,20 @@ static int balance_shallower(MemPage *pPage){ pChild->pgno, pPage->pgno)); } rc = reparentChildPages(pPage); + assert( pPage->nOverflow==0 ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + for(i=0; inCell; i++){ + Pgno ovfl = ovflPagePtr(pPage, findCell(pPage, i)); + if( ovfl ){ + rc = ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno); + if( rc!=SQLITE_OK ){ + goto end_shallow_balance; + } + } + } + } +#endif if( rc!=SQLITE_OK ) goto end_shallow_balance; releasePage(pChild); } @@ -4232,6 +4354,18 @@ static int balance_deeper(MemPage *pPage){ zeroPage(pPage, pChild->aData[0] & ~PTF_LEAF); put4byte(&pPage->aData[pPage->hdrOffset+8], pgnoChild); TRACE(("BALANCE: copy root %d into %d\n", pPage->pgno, pChild->pgno)); + if( pBt->autoVacuum ){ + int i; + rc = ptrmapPut(pBt, pChild->pgno, PTRMAP_BTREE, pPage->pgno); + if( rc ) return rc; + for(i=0; inCell; i++){ + Pgno pgno = ovflPagePtr(pChild, findOverflowCell(pChild, i)); + if( pgno ){ + rc = ptrmapPut(pBt, pgno, PTRMAP_OVERFLOW1, pChild->pgno); + if( rc ) return rc; + } + } + } rc = balance_nonroot(pChild); releasePage(pChild); return rc; diff --git a/test/autovacuum.test b/test/autovacuum.test index 69c2dfa103..ffa2e4edbf 100644 --- a/test/autovacuum.test +++ b/test/autovacuum.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the SELECT statement. # -# $Id: autovacuum.test,v 1.13 2004/11/22 05:26:28 danielk1977 Exp $ +# $Id: autovacuum.test,v 1.14 2005/01/15 12:45:51 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -60,6 +60,7 @@ lappend delete_orders {{1 2 3 4 5 6 7 8 9 10} {11 12 13 14 15 16 17 18 19 20}} lappend delete_orders {{19 8 17 15} {16 11 9 14} {18 5 3 1} {13 20 7 2} {6 12}} # The length of each table entry. +# set ENTRY_LEN 3500 set ENTRY_LEN 3500 do_test autovacuum-1.1 { diff --git a/test/autovacuum_ioerr2.test b/test/autovacuum_ioerr2.test index 9817f6d353..584da274a3 100644 --- a/test/autovacuum_ioerr2.test +++ b/test/autovacuum_ioerr2.test @@ -15,7 +15,7 @@ # The tests in this file use special facilities that are only # available in the SQLite test fixture. # -# $Id: autovacuum_ioerr2.test,v 1.1 2005/01/11 10:25:07 danielk1977 Exp $ +# $Id: autovacuum_ioerr2.test,v 1.2 2005/01/15 12:45:51 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -77,8 +77,7 @@ for {set n 1} {$go} {incr n} { } } execsql COMMIT - expr [file size test.db]/1024 - } {24} + } {} do_test autovacuum-ioerr2-2.$n.2 [subst { set ::sqlite_io_error_pending $n }] $n diff --git a/test/crash.test b/test/crash.test index 3d0751518b..23865a2868 100644 --- a/test/crash.test +++ b/test/crash.test @@ -20,7 +20,7 @@ # The special crash-test module with its os_test.c backend only works # on Unix. # -# $Id: crash.test,v 1.14 2005/01/13 11:07:54 danielk1977 Exp $ +# $Id: crash.test,v 1.15 2005/01/15 12:45:51 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -174,8 +174,8 @@ do_test crash-2.1 { execsql { SELECT sum(a), sum(b), sum(c) from abc } } {499500.0 999000.0 1498500.0} do_test crash-2.2 { - expr [file size test.db] / 1024 -} [expr $AUTOVACUUM ? 20 : 19] + expr ([file size test.db] / 1024)>16 +} {1} do_test crash-2.3 { crashsql 2 test.db-journal { DELETE FROM abc WHERE a < 800; @@ -214,8 +214,8 @@ do_test crash-3.0 { INSERT INTO abc SELECT * FROM abc; INSERT INTO abc SELECT * FROM abc; } - expr [file size test.db] / 1024 -} [expr $AUTOVACUUM ? 557 : 554] + expr ([file size test.db] / 1024) > 450 +} {1} for {set i 1} {$i < $repeats} {incr i} { set sig [signature] do_test crash-3.$i.1 { @@ -243,10 +243,6 @@ for {set i 1} {$i < $repeats} {incr i} { # crash-4.3.*: Test recovery when crash occurs during sync() of the master # journal file. # -set filesize 559 -ifcapable default_autovacuum { - set filesize 562 -} do_test crash-4.0 { file delete -force test2.db file delete -force test2.db-journal @@ -255,8 +251,8 @@ do_test crash-4.0 { PRAGMA aux.default_cache_size = 10; CREATE TABLE aux.abc2 AS SELECT 2*a as a, 2*b as b, 2*c as c FROM abc; } - expr [file size test2.db] / 1024 -} $filesize + expr ([file size test2.db] / 1024) > 450 +} {1} for {set i 1} {$i<$repeats} {incr i} { set sig [signature]