diff --git a/manifest b/manifest index 182bc50182..dc6753567c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sBTree\stests\s(CVS\s233) -D 2001-07-01T22:12:01 +C BTree\sand\spager\sare\sworking\spretty\swell\snow.\s(CVS\s234) +D 2001-07-02T17:51:46 F COPYRIGHT 74a8a6531a42e124df07ab5599aad63870fa0bd4 F Makefile.in df14e0f23d6946304d4681c24799d1ece965bf74 F README 51f6a4e7408b34afa5bc1c0485f61b6a4efb6958 @@ -12,8 +12,8 @@ F notes/notes1.txt b7c0812b704a022e88c621146ae50955c923d464 F notes/notes2.txt 7e3fafd5e25906c1fe1e95f13b089aa398ca403e F notes/notes3.txt 985bf688b59f1f52bfe6e4b1f896efdeffac1432 F src/TODO 38a68a489e56e9fd4a96263e0ff9404a47368ad4 -F src/btree.c 7e39906a52592d3683552235c2a7d3782cc9e6f9 -F src/btree.h 987d80658ae67f0e4d8b849539c113d4f9a7e835 +F src/btree.c d6bbe3152ce3eb47ffd0c797897bf75c5ca784fc +F src/btree.h 5fb5799bcb39900386ce6cae61fa33e357851ffe F src/build.c 4f6a2d551c56342cd4a0420654835be3ad179651 F src/dbbe.c b18259f99d87240cbe751021cf14dd3aa83a48af F src/dbbe.h 7235b15c6c5d8be0c4da469cef9620cee70b1cc8 @@ -32,8 +32,8 @@ F src/expr.c c4c24c3af1eba094a816522eb0e085bed518ee16 F src/insert.c aa528e20a787af85432a61daaea6df394bd251d7 F src/main.c 0a13c7a2beb8ce36aee43daf8c95989b200727a7 F src/md5.c 52f677bfc590e09f71d07d7e327bd59da738d07c -F src/pager.c 3e864a3e6cdec6f000a343f793360b42714028d8 -F src/pager.h d85259a2fd59e39f976abfb2bf6703c6f810e993 +F src/pager.c fbb1f1d8d2fd71333dfb9014852fd60194320732 +F src/pager.h ee84c00ca56ff6f0c53bbf216ede342cc99c701a F src/parse.y 8fc096948994a7ffbf61ba13129cc589f794a9cb F src/printf.c b1e22a47be8cdf707815647239991e08e8cb69f9 F src/random.c b36c3f57dc80c8f354e6bfbf39cf1e1de021d54a @@ -46,7 +46,7 @@ F src/table.c adcaf074f6c1075e86359174e68701fa2acfc4d6 F src/tclsqlite.c 7acb8887c44622214edb0dedeaab2593a3f86c62 F src/test1.c abb3cb427e735ae87e6533f5b3b7164b7da91bc4 F src/test2.c 0183625225a860397b4fd3041aefb48f77e4630a -F src/test3.c ad8ff3513c3deb2d3909eca0f94527017b6d2fe6 +F src/test3.c e13021bfd3bbbe6ec6c26d1de15d1a86ba5aae44 F src/tokenize.c 0118b57702cb6550769316e8443b06760b067acf F src/update.c 0cf789656a936d4356668393267692fa4b03ffc6 F src/util.c 1b396ac34e30dd6222d82e996c17b161bbc906bc @@ -54,8 +54,8 @@ F src/vdbe.c f93be4414ba892df9c5589815d2a57c1fb12c820 F src/vdbe.h dc1205da434c6a9da03b5d6b089270bbc8e6d437 F src/where.c 0c542fc44bd85152dfb8507862cfe2e60c629e9f F test/all.test 21d55a97e39e7ec5776751dc9dd8b1b51ef4a048 -F test/btree.test 2463425e01ef94ec123fdbfb0dcae33f5303d5b1 -F test/btree2.test 480e39c80109280cdfdbc305b77919c5eae69b2e +F test/btree.test 084f03bfc05551baff13b5e6ba61713d31e5621d +F test/btree2.test 485210a30a8efaf629bdc5d923918bbce6fae658 F test/copy.test b77a1214bd7756f2849d5c4fa6e715c0ff0c34eb F test/dbbe.test a022fe2d983848f786e17ef1fc6809cfd37fb02c F test/delete.test 50b9b1f06c843d591741dba7869433a105360dbf @@ -110,7 +110,7 @@ F www/opcode.tcl cb3a1abf8b7b9be9f3a228d097d6bf8b742c2b6f F www/sqlite.tcl cb0d23d8f061a80543928755ec7775da6e4f362f F www/tclsqlite.tcl 06f81c401f79a04f2c5ebfb97e7c176225c0aef2 F www/vdbe.tcl 0c8aaa529dd216ccbf7daaabd80985e413d5f9ad -P 42486880ed31318ef36a8831b959e9115b4fbac6 -R 453a76da442402a2188baa6e8a5a2ed6 +P 55c89bfdd35f1ad494618a451f9a1ed08502ae07 +R 3d46d052f2a18aacc3a120e98e99a6ad U drh -Z 41c451abad8201b164f89425654bbd84 +Z 32466dc4499f5824fce13920537543bc diff --git a/manifest.uuid b/manifest.uuid index d71cf0d6b6..d394676b55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55c89bfdd35f1ad494618a451f9a1ed08502ae07 \ No newline at end of file +a84fb078baf96dbfb5983981127dfc905074b7f9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ada01d9747..46d2332a1e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -21,7 +21,7 @@ ** http://www.hwaci.com/drh/ ** ************************************************************************* -** $Id: btree.c,v 1.19 2001/07/01 22:12:01 drh Exp $ +** $Id: btree.c,v 1.20 2001/07/02 17:51:46 drh Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -198,6 +198,12 @@ struct CellHdr { */ #define MX_CELL ((SQLITE_PAGE_SIZE-sizeof(PageHdr))/MIN_CELL_SIZE) +/* +** The amount of usable space on a single page of the BTree. This is the +** page size minus the overhead of the page header. +*/ +#define USABLE_SPACE (SQLITE_PAGE_SIZE - sizeof(PageHdr)) + /* ** The maximum amount of payload (in bytes) that can be stored locally for ** a database entry. If the entry contains more data than this, the @@ -205,8 +211,7 @@ struct CellHdr { ** ** This number is chosen so that at least 4 cells will fit on every page. */ -#define MX_LOCAL_PAYLOAD \ - (((SQLITE_PAGE_SIZE-sizeof(PageHdr))/4-(sizeof(CellHdr)+sizeof(Pgno)))&~3) +#define MX_LOCAL_PAYLOAD ((USABLE_SPACE/4-(sizeof(CellHdr)+sizeof(Pgno)))&~3) /* ** Data on a database page is stored as a linked list of Cell structures. @@ -361,6 +366,7 @@ static void defragmentPage(MemPage *pPage){ FreeBlk *pFBlk; char newPage[SQLITE_PAGE_SIZE]; + assert( sqlitepager_iswriteable(pPage) ); pc = sizeof(PageHdr); pPage->u.hdr.firstCell = pc; memcpy(newPage, pPage->u.aDisk, pc); @@ -409,6 +415,7 @@ static int allocateSpace(MemPage *pPage, int nByte){ int start; int cnt = 0; + assert( sqlitepager_iswriteable(pPage) ); assert( nByte==ROUNDUP(nByte) ); if( pPage->nFreeisOverfull ) return 0; pIdx = &pPage->u.hdr.firstFree; @@ -454,6 +461,7 @@ static void freeSpace(MemPage *pPage, int start, int size){ FreeBlk *pNew; FreeBlk *pNext; + assert( sqlitepager_iswriteable(pPage) ); assert( size == ROUNDUP(size) ); assert( start == ROUNDUP(start) ); pIdx = &pPage->u.hdr.firstFree; @@ -518,7 +526,7 @@ static int initPage(MemPage *pPage, Pgno pgnoThis, MemPage *pParent){ if( pPage->isInit ) return SQLITE_OK; pPage->isInit = 1; pPage->nCell = 0; - freeSpace = SQLITE_PAGE_SIZE - sizeof(PageHdr); + freeSpace = USABLE_SPACE; idx = pPage->u.hdr.firstCell; while( idx!=0 ){ if( idx>SQLITE_PAGE_SIZE-MIN_CELL_SIZE ) goto page_format_error; @@ -560,6 +568,7 @@ page_format_error: static void zeroPage(MemPage *pPage){ PageHdr *pHdr; FreeBlk *pFBlk; + assert( sqlitepager_iswriteable(pPage) ); memset(pPage, 0, SQLITE_PAGE_SIZE); pHdr = &pPage->u.hdr; pHdr->firstCell = 0; @@ -593,7 +602,12 @@ static void pageDestructor(void *pData){ ** for accessing the database. We do not open the database file ** until the first page is loaded. */ -int sqliteBtreeOpen(const char *zFilename, int mode, Btree **ppBtree){ +int sqliteBtreeOpen( + const char *zFilename, /* Name of the file containing the BTree database */ + int mode, /* Not currently used */ + int nCache, /* How many pages in the page cache */ + Btree **ppBtree /* Pointer to new Btree object written here */ +){ Btree *pBt; int rc; @@ -602,7 +616,8 @@ int sqliteBtreeOpen(const char *zFilename, int mode, Btree **ppBtree){ *ppBtree = 0; return SQLITE_NOMEM; } - rc = sqlitepager_open(&pBt->pPager, zFilename, 100, EXTRA_SIZE); + if( nCache<10 ) nCache = 10; + rc = sqlitepager_open(&pBt->pPager, zFilename, nCache, EXTRA_SIZE); if( rc!=SQLITE_OK ){ if( pBt->pPager ) sqlitepager_close(pBt->pPager); sqliteFree(pBt); @@ -1071,10 +1086,9 @@ static int moveToChild(BtCursor *pCur, int newPgno){ MemPage *pNewPage; rc = sqlitepager_get(pCur->pBt->pPager, newPgno, (void**)&pNewPage); - if( rc ){ - return rc; - } - initPage(pNewPage, newPgno, pCur->pPage); + if( rc ) return rc; + rc = initPage(pNewPage, newPgno, pCur->pPage); + if( rc ) return rc; sqlitepager_unref(pCur->pPage); pCur->pPage = pNewPage; pCur->idx = 0; @@ -1118,6 +1132,8 @@ static int moveToRoot(BtCursor *pCur){ rc = sqlitepager_get(pCur->pBt->pPager, pCur->pgnoRoot, (void**)&pNew); if( rc ) return rc; + rc = initPage(pNew, pCur->pgnoRoot, 0); + if( rc ) return rc; sqlitepager_unref(pCur->pPage); pCur->pPage = pNew; pCur->idx = 0; @@ -1437,7 +1453,7 @@ static void reparentPage(Pager *pPager, Pgno pgno, MemPage *pNewParent){ if( pgno==0 ) return; assert( pPager!=0 ); pThis = sqlitepager_lookup(pPager, pgno); - if( pThis ){ + if( pThis && pThis->isInit ){ if( pThis->pParent!=pNewParent ){ if( pThis->pParent ) sqlitepager_unref(pThis->pParent); pThis->pParent = pNewParent; @@ -1480,6 +1496,7 @@ static void dropCell(MemPage *pPage, int idx, int sz){ int j; assert( idx>=0 && idxnCell ); assert( sz==cellSize(pPage->apCell[idx]) ); + assert( sqlitepager_iswriteable(pPage) ); freeSpace(pPage, Addr(pPage->apCell[idx]) - Addr(pPage), sz); for(j=idx; jnCell-1; j++){ pPage->apCell[j] = pPage->apCell[j+1]; @@ -1504,6 +1521,7 @@ static void insertCell(MemPage *pPage, int i, Cell *pCell, int sz){ int idx, j; assert( i>=0 && i<=pPage->nCell ); assert( sz==cellSize(pCell) ); + assert( sqlitepager_iswriteable(pPage) ); idx = allocateSpace(pPage, sz); for(j=pPage->nCell; j>i; j--){ pPage->apCell[j] = pPage->apCell[j-1]; @@ -1527,6 +1545,7 @@ static void insertCell(MemPage *pPage, int i, Cell *pCell, int sz){ static void relinkCellList(MemPage *pPage){ int i; u16 *pIdx; + assert( sqlitepager_iswriteable(pPage) ); pIdx = &pPage->u.hdr.firstCell; for(i=0; inCell; i++){ int idx = Addr(pPage->apCell[i]) - Addr(pPage); @@ -1620,9 +1639,10 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ int nxDiv; /* Next divider slot in pParent->apCell[] */ int rc; /* The return code */ int iCur; /* apCell[iCur] is the cell of the cursor */ - int usedPerPage; /* Memory needed for each page */ - int freePerPage; /* Average free space per page */ int totalSize; /* Total bytes for all cells */ + int subtotal; /* Subtotal of bytes in cells on one page */ + int cntNew[4]; /* Index in apCell[] of cell after i-th page */ + int szNew[4]; /* Combined size of cells place on i-th page */ MemPage *extraUnref = 0; /* A page that needs to be unref-ed */ Pgno pgno; /* Page number */ Cell *apCell[MX_CELL*3+5]; /* All cells from pages being balanceed */ @@ -1634,7 +1654,8 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ ** Return without doing any work if pPage is neither overfull nor ** underfull. */ - if( !pPage->isOverfull && pPage->nFreeisOverfull && pPage->nFreeu.hdr.rightChild; rc = sqlitepager_get(pBt->pPager, pgnoChild, (void**)&pChild); if( rc ) return rc; memcpy(pPage, pChild, SQLITE_PAGE_SIZE); pPage->isInit = 0; - initPage(pPage, sqlitepager_pagenumber(pPage), 0); + rc = initPage(pPage, sqlitepager_pagenumber(pPage), 0); + assert( rc==SQLITE_OK ); reparentChildPages(pBt->pPager, pPage); freePage(pBt, pChild, pgnoChild); sqlitepager_unref(pChild); @@ -1689,6 +1709,7 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ if( rc ) return rc; rc = allocatePage(pBt, &pChild, &pgnoChild); if( rc ) return rc; + assert( sqlitepager_iswriteable(pChild) ); copyPage(pChild, pPage); pChild->pParent = pPage; sqlitepager_ref(pPage); @@ -1703,10 +1724,9 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ pPage->u.hdr.rightChild = pgnoChild; pParent = pPage; pPage = pChild; - }else{ - rc = sqlitepager_write(pPage); - if( rc ) return rc; } + rc = sqlitepager_write(pParent); + if( rc ) return rc; /* ** Find the Cell in the parent page whose h.leftChild points back @@ -1762,6 +1782,8 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ } rc = sqlitepager_get(pBt->pPager, pgnoOld[i], (void**)&apOld[i]); if( rc ) goto balance_cleanup; + rc = initPage(apOld[i], pgnoOld[i], pParent); + if( rc ) goto balance_cleanup; nOld++; } @@ -1818,28 +1840,50 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ } /* - ** Estimate the number of pages needed. Record this number in "k" - ** for now. It will get transferred to nNew as we allocate the - ** new pages. + ** Figure out the number of pages needed to hold all nCell cells. + ** Store this number in "k". Also compute szNew[] which is the total + ** size of all cells on the i-th page and cntNew[] which is the index + ** in apCell[] of the cell that divides path i from path i+1. + ** cntNew[k] should equal nCell. + ** + ** This little patch of code is critical for keeping the tree + ** balanced. */ totalSize = 0; for(i=0; i USABLE_SPACE ){ + szNew[k] = subtotal - szCell[i]; + cntNew[k] = i; + subtotal = 0; + k++; + } + } + szNew[k] = subtotal; + cntNew[k] = nCell; + k++; + for(i=k-1; i>0; i--){ + while( szNew[i]0 ); + szNew[i] += szCell[cntNew[i-1]]; + szNew[i-1] -= szCell[cntNew[i-1]-1]; + } + } + assert( cntNew[0]>0 ); /* - ** Allocate new pages + ** Allocate k new pages */ for(i=0; iisInit = 1; } /* @@ -1849,11 +1893,13 @@ static int balance(Btree *pBt, MemPage *pPage, BtCursor *pCur){ j = 0; for(i=0; inFree>freePerPage && szCell[j]<=pNew->nFree ){ + while( jnFree>=szCell[j] ); if( pCur && iCur==j ){ pCur->pPage = pNew; pCur->idx = pNew->nCell; } insertCell(pNew, pNew->nCell, apCell[j], szCell[j]); j++; } + assert( pNew->nCell>0 ); assert( !pNew->isOverfull ); relinkCellList(pNew); if( iu.hdr.rightChild = apOld[nOld-1]->u.hdr.rightChild; if( nxDiv==pParent->nCell ){ pParent->u.hdr.rightChild = pgnoNew[nNew-1]; @@ -1999,6 +2046,8 @@ int sqliteBtreeDelete(BtCursor *pCur){ if( rc!=SQLITE_OK ){ return SQLITE_CORRUPT; } + rc = sqlitepager_write(leafCur.pPage); + if( rc ) return rc; dropCell(pPage, pCur->idx, cellSize(pCell)); pNext = leafCur.pPage->apCell[leafCur.idx]; szNext = cellSize(pNext); @@ -2012,8 +2061,12 @@ int sqliteBtreeDelete(BtCursor *pCur){ releaseTempCursor(&leafCur); }else{ dropCell(pPage, pCur->idx, cellSize(pCell)); + if( pCur->idx>=pPage->nCell && pCur->idx>0 ){ + pCur->idx--; + }else{ + pCur->bSkipNext = 1; + } rc = balance(pCur->pBt, pPage, pCur); - pCur->bSkipNext = 1; } return rc; } @@ -2031,7 +2084,7 @@ int sqliteBtreeCreateTable(Btree *pBt, int *piTable){ } rc = allocatePage(pBt, &pRoot, &pgnoRoot); if( rc ) return rc; - sqlitepager_write(pRoot); + assert( sqlitepager_iswriteable(pRoot) ); zeroPage(pRoot); sqlitepager_unref(pRoot); *piTable = (int)pgnoRoot; @@ -2050,6 +2103,8 @@ static int clearDatabasePage(Btree *pBt, Pgno pgno, int freePageFlag){ rc = sqlitepager_get(pBt->pPager, pgno, (void**)&pPage); if( rc ) return rc; + rc = sqlitepager_write(pPage); + if( rc ) return rc; idx = pPage->u.hdr.firstCell; while( idx>0 ){ pCell = (Cell*)&pPage->u.aDisk[idx]; @@ -2157,7 +2212,7 @@ int sqliteBtreeUpdateMeta(Btree *pBt, int *aMeta){ ** Print a disassembly of the given page on standard output. This routine ** is used for debugging and testing only. */ -int sqliteBtreePageDump(Btree *pBt, int pgno){ +int sqliteBtreePageDump(Btree *pBt, int pgno, int recursive){ int rc; MemPage *pPage; int i, j; @@ -2169,6 +2224,7 @@ int sqliteBtreePageDump(Btree *pBt, int pgno){ if( rc ){ return rc; } + if( recursive ) printf("PAGE %d:\n", pgno); i = 0; idx = pPage->u.hdr.firstCell; while( idx>0 && idx<=SQLITE_PAGE_SIZE-MIN_CELL_SIZE ){ @@ -2183,11 +2239,11 @@ int sqliteBtreePageDump(Btree *pBt, int pgno){ } payload[sz] = 0; printf( - "cell %2d: i=%-10s chld=%-4d nk=%-3d nd=%-3d payload=%s\n", + "cell %2d: i=%-10s chld=%-4d nk=%-4d nd=%-4d payload=%s\n", i, range, (int)pCell->h.leftChild, pCell->h.nKey, pCell->h.nData, payload ); - if( pPage->apCell[i]!=pCell ){ + if( pPage->isInit && pPage->apCell[i]!=pCell ){ printf("**** apCell[%d] does not match on prior entry ****\n", i); } i++; @@ -2212,6 +2268,15 @@ int sqliteBtreePageDump(Btree *pBt, int pgno){ if( idx!=0 ){ printf("ERROR: next freeblock index out of range: %d\n", idx); } + if( recursive && pPage->u.hdr.rightChild!=0 ){ + idx = pPage->u.hdr.firstCell; + while( idx>0 && idxu.aDisk[idx]; + sqliteBtreePageDump(pBt, pCell->h.leftChild, 1); + idx = pCell->h.iNext; + } + sqliteBtreePageDump(pBt, pPage->u.hdr.rightChild, 1); + } sqlitepager_unref(pPage); return SQLITE_OK; } @@ -2274,6 +2339,8 @@ struct SanityCheck { Pager *pPager; // The associated pager. Also accessible by pBt->pPager int nPage; // Number of pages in the database int *anRef; // Number of times each page is referenced + int nTreePage; // Number of BTree pages + int nByte; // Number of bytes of data stored on BTree pages char *zErrMsg; // An error message. NULL of no errors seen. }; @@ -2355,7 +2422,7 @@ static void checkList(SanityCheck *pCheck, int iPage, int N, char *zContext){ ** 5. Check the integrity of overflow pages. ** 6. Recursively call checkTreePage on all children. ** 7. Verify that the depth of all children is the same. -** 8. Make sure this page is at least 50% full or else it is +** 8. Make sure this page is at least 33% full or else it is ** the root of the tree. */ static int checkTreePage( @@ -2465,11 +2532,18 @@ static int checkTreePage( /* Check that free space is kept to a minimum */ - if( pParent && pPage->nFree>SQLITE_PAGE_SIZE/3 ){ +#if 0 + if( pParent && pParent->nCell>2 && pPage->nFree>3*SQLITE_PAGE_SIZE/4 ){ sprintf(zMsg, "free space (%d) greater than max (%d)", pPage->nFree, SQLITE_PAGE_SIZE/3); checkAppendMsg(pCheck, zContext, zMsg); } +#endif + + /* Update freespace totals. + */ + pCheck->nTreePage++; + pCheck->nByte += USABLE_SPACE - pPage->nFree; sqlitepager_unref(pPage); return depth; diff --git a/src/btree.h b/src/btree.h index a5de692f2c..6c54b67084 100644 --- a/src/btree.h +++ b/src/btree.h @@ -24,13 +24,13 @@ ** This header file defines the interface that the sqlite B-Tree file ** subsystem. ** -** @(#) $Id: btree.h,v 1.8 2001/06/30 21:53:53 drh Exp $ +** @(#) $Id: btree.h,v 1.9 2001/07/02 17:51:46 drh Exp $ */ typedef struct Btree Btree; typedef struct BtCursor BtCursor; -int sqliteBtreeOpen(const char *zFilename, int mode, Btree **ppBtree); +int sqliteBtreeOpen(const char *zFilename, int mode, int nPg, Btree **ppBtree); int sqliteBtreeClose(Btree*); int sqliteBtreeBeginTrans(Btree*); @@ -58,7 +58,7 @@ int sqliteBtreeUpdateMeta(Btree*, int*); #ifdef SQLITE_TEST -int sqliteBtreePageDump(Btree*, int); +int sqliteBtreePageDump(Btree*, int, int); int sqliteBtreeCursorDump(BtCursor*, int*); Pager *sqliteBtreePager(Btree*); char *sqliteBtreeSanityCheck(Btree*, int*, int); diff --git a/src/pager.c b/src/pager.c index ebd641f342..57866918a4 100644 --- a/src/pager.c +++ b/src/pager.c @@ -27,7 +27,7 @@ ** all writes in order to support rollback. Locking is used to limit ** access to one or more reader or one writer. ** -** @(#) $Id: pager.c,v 1.12 2001/06/28 01:54:49 drh Exp $ +** @(#) $Id: pager.c,v 1.13 2001/07/02 17:51:46 drh Exp $ */ #include "sqliteInt.h" #include "pager.h" @@ -122,6 +122,7 @@ struct Pager { int nHit, nMiss, nOvfl; /* Cache hits, missing, and LRU overflows */ unsigned char state; /* SQLITE_UNLOCK, _READLOCK or _WRITELOCK */ unsigned char errMask; /* One of several kinds of errors */ + unsigned char *aInJournal; /* One bit for each page in the database file */ PgHdr *pFirst, *pLast; /* List of free pages */ PgHdr *pAll; /* List of all pages */ PgHdr *aHash[N_PG_HASH]; /* Hash table to map page number of PgHdr */ @@ -210,6 +211,7 @@ static int pager_unlock(fd){ ** the beginning of the file. */ static int pager_seek(int fd, off_t whereto){ + /*printf("SEEK to page %d\n", whereto/SQLITE_PAGE_SIZE + 1);*/ lseek(fd, whereto, SEEK_SET); return SQLITE_OK; } @@ -232,6 +234,7 @@ static int pager_truncate(int fd, Pgno mxPg){ */ static int pager_read(int fd, void *pBuf, int nByte){ int rc; + /* printf("READ\n");*/ rc = read(fd, pBuf, nByte); if( rc<0 ){ memset(pBuf, 0, nByte); @@ -253,6 +256,7 @@ static int pager_read(int fd, void *pBuf, int nByte){ */ static int pager_write(int fd, const void *pBuf, int nByte){ int rc; + /*printf("WRITE\n");*/ rc = write(fd, pBuf, nByte); if( rczJournal); close(pPager->jfd); pPager->jfd = -1; + sqliteFree( pPager->aInJournal ); + pPager->aInJournal = 0; for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ pPg->inJournal = 0; pPg->dirty = 0; @@ -434,6 +440,7 @@ static int pager_playback(Pager *pPager){ pPg = pager_lookup(pPager, pgRec.pgno); if( pPg ){ memcpy(PGHDR_TO_DATA(pPg), pgRec.aData, SQLITE_PAGE_SIZE); + memset(PGHDR_TO_EXTRA(pPg), 0, pPager->nExtra); } rc = pager_seek(pPager->fd, (pgRec.pgno-1)*SQLITE_PAGE_SIZE); if( rc!=SQLITE_OK ) break; @@ -719,9 +726,9 @@ int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){ /* Recycle an older page. First locate the page to be recycled. ** Try to find one that is not dirty and is near the head of ** of the free list */ - int cnt = 4; + int cnt = pPager->mxPage/2; pPg = pPager->pFirst; - while( pPg->dirty && 0dirty && 0pNextFree ){ pPg = pPg->pNextFree; } if( pPg==0 || pPg->dirty ) pPg = pPager->pFirst; @@ -752,12 +759,19 @@ int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){ /* Unlink the old page from the free list and the hash table */ - pPager->pFirst = pPg->pNextFree; - if( pPager->pFirst ){ - pPager->pFirst->pPrevFree = 0; + if( pPg->pPrevFree ){ + pPg->pPrevFree->pNextFree = pPg->pNextFree; }else{ - pPager->pLast = 0; + assert( pPager->pFirst==pPg ); + pPager->pFirst = pPg->pNextFree; } + if( pPg->pNextFree ){ + pPg->pNextFree->pPrevFree = pPg->pPrevFree; + }else{ + assert( pPager->pLast==pPg ); + pPager->pLast = pPg->pPrevFree; + } + pPg->pNextFree = pPg->pPrevFree = 0; if( pPg->pNextHash ){ pPg->pNextHash->pPrevHash = pPg->pPrevHash; } @@ -768,10 +782,15 @@ int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){ assert( pPager->aHash[h]==pPg ); pPager->aHash[h] = pPg->pNextHash; } + pPg->pNextHash = pPg->pPrevHash = 0; pPager->nOvfl++; } pPg->pgno = pgno; - pPg->inJournal = 0; + if( pPager->aInJournal && pgno<=pPager->origDbSize ){ + pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0; + }else{ + pPg->inJournal = 0; + } pPg->dirty = 0; pPg->nRef = 1; REFINFO(pPg); @@ -910,6 +929,11 @@ int sqlitepager_write(void *pData){ if( pPg->inJournal ){ return SQLITE_OK; } assert( pPager->state!=SQLITE_UNLOCK ); if( pPager->state==SQLITE_READLOCK ){ + assert( pPager->aInJournal==0 ); + pPager->aInJournal = sqliteMalloc( pPager->dbSize/8 + 1 ); + if( pPager->aInJournal==0 ){ + return SQLITE_NOMEM; + } pPager->jfd = open(pPager->zJournal, O_RDWR|O_CREAT, 0644); if( pPager->jfd<0 ){ return SQLITE_CANTOPEN; @@ -952,6 +976,8 @@ int sqlitepager_write(void *pData){ pPager->errMask |= PAGER_ERR_FULL; return rc; } + assert( pPager->aInJournal!=0 ); + pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7); } pPg->inJournal = 1; if( pPager->dbSizepgno ){ @@ -960,6 +986,16 @@ int sqlitepager_write(void *pData){ return rc; } +/* +** Return TRUE if the page given in the argument was previous passed +** to sqlitepager_write(). In other words, return TRUE if it is ok +** to change the content of the page. +*/ +int sqlitepager_iswriteable(void *pData){ + PgHdr *pPg = DATA_TO_PGHDR(pData); + return pPg->dirty; +} + /* ** Commit all changes to the database and release the write lock. ** diff --git a/src/pager.h b/src/pager.h index 208d898c0a..1fefbec461 100644 --- a/src/pager.h +++ b/src/pager.h @@ -25,7 +25,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.6 2001/06/28 01:54:49 drh Exp $ +** @(#) $Id: pager.h,v 1.7 2001/07/02 17:51:46 drh Exp $ */ /* @@ -53,6 +53,7 @@ int sqlitepager_ref(void*); int sqlitepager_unref(void*); Pgno sqlitepager_pagenumber(void*); int sqlitepager_write(void*); +int sqlitepager_iswriteable(void*); int sqlitepager_pagecount(Pager*); int sqlitepager_commit(Pager*); int sqlitepager_rollback(Pager*); diff --git a/src/test3.c b/src/test3.c index 0c135a74ca..27745079a8 100644 --- a/src/test3.c +++ b/src/test3.c @@ -25,7 +25,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test3.c,v 1.6 2001/07/01 22:12:02 drh Exp $ +** $Id: test3.c,v 1.7 2001/07/02 17:51:46 drh Exp $ */ #include "sqliteInt.h" #include "pager.h" @@ -79,7 +79,7 @@ static int btree_open( " FILENAME\"", 0); return TCL_ERROR; } - rc = sqliteBtreeOpen(argv[1], 0666, &pBt); + rc = sqliteBtreeOpen(argv[1], 0666, 10, &pBt); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, errorName(rc), 0); return TCL_ERROR; @@ -376,7 +376,37 @@ static int btree_page_dump( } if( Tcl_GetInt(interp, argv[1], (int*)&pBt) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &iPage) ) return TCL_ERROR; - rc = sqliteBtreePageDump(pBt, iPage); + rc = sqliteBtreePageDump(pBt, iPage, 0); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** Usage: btree_tree_dump ID PAGENUM +** +** Print a disassembly of a page and all its child pages on standard output +*/ +static int btree_tree_dump( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Btree *pBt; + int iPage; + int rc; + + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pBt) ) return TCL_ERROR; + if( Tcl_GetInt(interp, argv[2], &iPage) ) return TCL_ERROR; + rc = sqliteBtreePageDump(pBt, iPage, 1); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, errorName(rc), 0); return TCL_ERROR; @@ -795,6 +825,7 @@ int Sqlitetest3_Init(Tcl_Interp *interp){ Tcl_CreateCommand(interp, "btree_get_meta", btree_get_meta, 0, 0); Tcl_CreateCommand(interp, "btree_update_meta", btree_update_meta, 0, 0); Tcl_CreateCommand(interp, "btree_page_dump", btree_page_dump, 0, 0); + Tcl_CreateCommand(interp, "btree_tree_dump", btree_tree_dump, 0, 0); Tcl_CreateCommand(interp, "btree_pager_stats", btree_pager_stats, 0, 0); Tcl_CreateCommand(interp, "btree_pager_ref_dump", btree_pager_ref_dump, 0, 0); Tcl_CreateCommand(interp, "btree_cursor", btree_cursor, 0, 0); diff --git a/test/btree.test b/test/btree.test index 3108557d98..e630404b38 100644 --- a/test/btree.test +++ b/test/btree.test @@ -23,7 +23,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is btree database backend # -# $Id: btree.test,v 1.5 2001/06/30 21:53:53 drh Exp $ +# $Id: btree.test,v 1.6 2001/07/02 17:51:47 drh Exp $ set testdir [file dirname $argv0] @@ -718,6 +718,7 @@ do_test btree-8.23 { do_test btree-8.24 { lindex [btree_pager_stats $::b1] 1 } {2} +#btree_pager_ref_dump $::b1 # Check page splitting logic # @@ -728,7 +729,7 @@ do_test btree-9.1 { btree_insert $::c1 $key $data } } {} -#btree_page_dump $::b1 2 +#btree_tree_dump $::b1 2 #btree_pager_ref_dump $::b1 #set pager_refinfo_enable 1 do_test btree-9.2 { @@ -820,12 +821,15 @@ do_test btree-10.4 { btree_delete $::c1 select_keys $::c1 } {001 002 003 004 005 006 007 008 009 010 012 013 014 015 016 017 018 019 020} -#btree_page_dump $::b1 2 +#btree_tree_dump $::b1 2 +#btree_pager_ref_dump $::b1 for {set i 1} {$i<=20} {incr i} { do_test btree-10.5.$i { btree_move_to $::c1 [format %03d $i] lindex [btree_pager_stats $::b1] 1 } {2} + #btree_pager_ref_dump $::b1 + #btree_tree_dump $::b1 2 } # Create a tree with lots more pages @@ -886,8 +890,9 @@ do_test btree-11.4.3 { } {259} do_test btree-11.4.4 { btree_move_to $::c1 257 - btree_key $::c1 -} {256} + set n [btree_key $::c1] + expr {$n==256||$n==258} +} {1} do_test btree-11.5 { btree_move_to $::c1 513 btree_delete $::c1 @@ -908,8 +913,9 @@ do_test btree-11.5.3 { } {515} do_test btree-11.5.4 { btree_move_to $::c1 513 - btree_key $::c1 -} {512} + set n [btree_key $::c1] + expr {$n==512||$n==514} +} {1} do_test btree-11.6 { btree_move_to $::c1 769 btree_delete $::c1 @@ -930,8 +936,9 @@ do_test btree-11.6.3 { } {770} do_test btree-11.6.4 { btree_move_to $::c1 769 - btree_key $::c1 -} {768} + set n [btree_key $::c1] + expr {$n==768||$n==770} +} {1} #btree_page_dump $::b1 2 #btree_page_dump $::b1 25 diff --git a/test/btree2.test b/test/btree2.test index 0e569fb6fb..62874eb7e2 100644 --- a/test/btree2.test +++ b/test/btree2.test @@ -23,7 +23,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is btree database backend # -# $Id: btree2.test,v 1.1 2001/07/01 22:12:02 drh Exp $ +# $Id: btree2.test,v 1.2 2001/07/02 17:51:47 drh Exp $ set testdir [file dirname $argv0] @@ -43,6 +43,7 @@ if {$dbprefix!="memory:" && [info commands btree_open]!=""} { # An explanation for what all these tables are used for is provided below. # do_test btree2-1.1 { + expr srand(1) file delete -force test2.bt file delete -force test2.bt-journal set ::b [btree_open test2.bt] @@ -136,6 +137,12 @@ proc make_payload {keynum L len} { # success or an error message if something is amiss. # proc check_invariants {} { + set ck [btree_sanity_check $::b 2 3 4 5 6] + if {$ck!=""} { + puts "\n*** SANITY:\n$ck" + exit + return $ck + } btree_move_to $::c3 {} btree_move_to $::c4 {} btree_move_to $::c2 N @@ -145,12 +152,16 @@ proc check_invariants {} { set LM1 [expr {$L-1}] for {set i 1} {$i<=$N} {incr i} { set key [btree_key $::c3] - scan $key %d k + if {[scan $key %d k]<1} {set k 0} if {$k!=$i} { set key [btree_key $::c4] - scan $key %d k + if {[scan $key %d k]<1} {set k 0} if {$k!=$i} { - return "Key $i is missing from both foreground and backgroun" + # puts "MISSING $i" + # puts {Page 3:}; btree_page_dump $::b 3 + # puts {Page 4:}; btree_page_dump $::b 4 + # exit + return "Key $i is missing from both foreground and background" } set data [btree_data $::c4] btree_next $::c4 @@ -193,7 +204,9 @@ proc check_invariants {} { # $I and it is put in background with probability (1.0-$I). It gets # a long key with probability $K and long data with probability $D. # +set chngcnt 0 proc random_changes {n I K D} { + btree_move_to $::c2 N set N [btree_data $::c2] btree_move_to $::c2 L set L [btree_data $::c2] @@ -201,38 +214,51 @@ proc random_changes {n I K D} { set total [expr {int($N*$n)}] set format %0${L}d for {set i 0} {$i<$total} {incr i} { - set k [expr {int(rand()*$N)}] + set k [expr {int(rand()*$N)+1}] set insert [expr {rand()<=$I}] set longkey [expr {rand()<=$K}] set longdata [expr {rand()<=$D}] + # incr ::chngcnt + # if {$::chngcnt==251} {btree_tree_dump $::b 3} + # puts "CHANGE $::chngcnt: $k $insert $longkey $longdata" if {$longkey} { set x [expr {rand()}] - set keylen [expr {int($x*$x*$x*$x*3000)}] + set keylen [expr {int($x*$x*$x*$x*3000)+10}] } else { set keylen $L } set key [make_payload $k $L $keylen] if {$longdata} { set x [expr {rand()}] - set datalen [expr {int($x*$x*$x*$x*3000)}] + set datalen [expr {int($x*$x*$x*$x*3000)+10}] } else { set datalen $L } set data [make_payload $k $L $datalen] set basekey [format $format $k] - if {$insert} { - btree_move_to $::c4 $basekey - if {[scan [btree_key $::c4] %d kx]<1} {set kx -1} - if {$kx==$k} { - btree_delete $::c4 - } - btree_insert $::c3 $key $data + if {[set c [btree_move_to $::c3 $basekey]]==0} { + btree_delete $::c3 } else { - btree_move_to $::c3 $basekey - if {[scan [btree_key $::c4] %d kx]<1} {set kx -1} - if {$kx==$k} { + if {$c<0} {btree_next $::c3} + if {[string match $basekey* [btree_key $::c3]]} { btree_delete $::c3 } + } + if {[set c [btree_move_to $::c4 $basekey]]==0} { + btree_delete $::c4 + } else { + if {$c<0} {btree_next $::c4} + if {[string match $basekey* [btree_key $::c4]]} { + btree_delete $::c4 + } + } + if {[scan [btree_key $::c4] %d kx]<1} {set kx -1} + if {$kx==$k} { + btree_delete $::c4 + } + if {$insert} { + btree_insert $::c3 $key $data + } else { btree_insert $::c4 $key $data } if {$longkey} { @@ -245,8 +271,14 @@ proc random_changes {n I K D} { } elseif {[btree_move_to $::c6 $basekey]==0} { btree_delete $::c6 } + # set ck [btree_sanity_check $::b 2 3 4 5 6] + # if {$ck!=""} { + # puts "\nSANITY CHECK FAILED!\n$ck" + # exit + # } + # puts "PAGE 3:"; btree_page_dump $::b 3 + # puts "PAGE 4:"; btree_page_dump $::b 4 } - return [btree_sanity_check $::b 2 3 4 5 6] } # Repeat this test sequence on database of various sizes @@ -254,6 +286,9 @@ proc random_changes {n I K D} { set testno 2 foreach {N L} { 10 2 + 50 2 + 200 3 + 2000 5 } { puts "**** N=$N L=$L ****" set hash [md5file test2.bt] @@ -304,6 +339,11 @@ foreach {N L} { do_test btree2-$testno.7 { btree_close $::b set ::b [btree_open test2.bt] + set ::c2 [btree_cursor $::b 2] + set ::c3 [btree_cursor $::b 3] + set ::c4 [btree_cursor $::b 4] + set ::c5 [btree_cursor $::b 5] + set ::c6 [btree_cursor $::b 6] check_invariants } {} @@ -311,24 +351,32 @@ foreach {N L} { # set num2 1 foreach {n I K D} { - 0.5 0.5 0.5 0.5 + 0.5 0.5 0.1 0.1 + 1.0 0.2 0.1 0.1 + 1.0 0.8 0.1 0.1 + 2.0 0.0 0.1 0.1 + 2.0 1.0 0.1 0.1 + 2.0 0.0 0.0 0.0 + 2.0 1.0 0.0 0.0 } { set testid btree2-$testno.8.$num2 + set cnt 6 + for {set i 2} {$i<=6} {incr i} { + if {[lindex [btree_cursor_dump [set ::c$i]] 0]!=$i} {incr cnt} + } do_test $testid.1 { - set ::c2 [btree_cursor $::b 2] - set ::c3 [btree_cursor $::b 3] - set ::c4 [btree_cursor $::b 4] - set ::c5 [btree_cursor $::b 5] - set ::c6 [btree_cursor $::b 6] btree_begin_transaction $::b lindex [btree_pager_stats $::b] 1 - } {6} + } $cnt set hash [md5file test2.bt] - do_test $testid.2 [subst -nocommands { + # exec cp test2.bt test2.bt.bu1 + do_test $testid.2 [subst { random_changes $n $I $K $D - check_invariants }] {} do_test $testid.3 { + check_invariants + } {} + do_test $testid.4 { btree_close_cursor $::c2 btree_close_cursor $::c3 btree_close_cursor $::c4 @@ -337,33 +385,49 @@ foreach {N L} { btree_rollback $::b md5file test2.bt } $hash - do_test $testid.4 [subst -nocommands { - btree_begin_transaction $::b - set ::c2 [btree_cursor $::b 2] - set ::c3 [btree_cursor $::b 3] - set ::c4 [btree_cursor $::b 4] - set ::c5 [btree_cursor $::b 5] - set ::c6 [btree_cursor $::b 6] + # exec cp test2.bt test2.bt.bu2 + btree_begin_transaction $::b + set ::c2 [btree_cursor $::b 2] + set ::c3 [btree_cursor $::b 3] + set ::c4 [btree_cursor $::b 4] + set ::c5 [btree_cursor $::b 5] + set ::c6 [btree_cursor $::b 6] + do_test $testid.5 [subst { random_changes $n $I $K $D - check_invariants }] {} - do_test $testid.5 { + do_test $testid.6 { + check_invariants + } {} + do_test $testid.7 { btree_commit $::b check_invariants } {} set hash [md5file test2.bt] - do_test $testid.6 { + do_test $testid.8 { btree_close_cursor $::c2 btree_close_cursor $::c3 btree_close_cursor $::c4 btree_close_cursor $::c5 btree_close_cursor $::c6 + lindex [btree_pager_stats $::b] 1 + } {0} + do_test $testid.9 { btree_close $::b set ::b [btree_open test2.bt] + set ::c2 [btree_cursor $::b 2] + set ::c3 [btree_cursor $::b 3] + set ::c4 [btree_cursor $::b 4] + set ::c5 [btree_cursor $::b 5] + set ::c6 [btree_cursor $::b 6] check_invariants } {} incr num2 } + btree_close_cursor $::c2 + btree_close_cursor $::c3 + btree_close_cursor $::c4 + btree_close_cursor $::c5 + btree_close_cursor $::c6 incr testno }