diff --git a/Makefile.in b/Makefile.in index 3428df3f6c..c317a99c4c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -48,8 +48,8 @@ LIBREADLINE = @TARGET_READLINE_LIBS@ # Object files for the SQLite library. # LIBOBJ = build.o dbbe.o dbbegdbm.o dbbemem.o delete.o expr.o insert.o \ - main.o parse.o printf.o random.o select.o table.o tokenize.o \ - update.o util.o vdbe.o where.o tclsqlite.o + main.o pager.o parse.o printf.o random.o select.o table.o \ + tokenize.o update.o util.o vdbe.o where.o tclsqlite.o # All of the source code files. # @@ -63,6 +63,7 @@ SRC = \ $(TOP)/src/expr.c \ $(TOP)/src/insert.c \ $(TOP)/src/main.c \ + $(TOP)/src/pager.c \ $(TOP)/src/parse.y \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ @@ -79,6 +80,12 @@ SRC = \ $(TOP)/src/vdbe.h \ $(TOP)/src/where.c +# Source code to the test files. +# +TESTSRC = \ + $(TOP)/src/test1.c \ + $(TOP)/src/test2.c + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # @@ -182,6 +189,9 @@ tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) printf.o: $(TOP)/src/printf.c $(HDR) $(TCC) $(GDBM_FLAGS) $(TCL_FLAGS) -c $(TOP)/src/printf.c +pager.o: $(TOP)/src/pager.c $(HDR) + $(TCC) $(GDBM_FLAGS) $(TCL_FLAGS) -c $(TOP)/src/pager.c + gdbmdump: $(TOP)/tool/gdbmdump.c $(TCC) $(GDBM_FLAGS) -o gdbmdump $(TOP)/tool/gdbmdump.c $(LIBGDBM) @@ -189,9 +199,9 @@ tclsqlite: $(TOP)/src/tclsqlite.c libsqlite.a $(TCC) $(TCL_FLAGS) -DTCLSH=1 -o sqlite_tester \ $(TOP)/src/tclsqlite.c libsqlite.a $(LIBGDBM) $(LIBTCL) -sqlite_tester: $(TOP)/src/tclsqlite.c libsqlite.a $(TOP)/src/test1.c - $(TCC) $(TCL_FLAGS) -DTCLSH=1 -DSQLITE_TEST1=1 -o sqlite_tester \ - $(TOP)/src/test1.c $(TOP)/src/tclsqlite.c \ +sqlite_tester: $(TOP)/src/tclsqlite.c libsqlite.a $(TESTSRC) + $(TCC) $(TCL_FLAGS) -DTCLSH=1 -DSQLITE_TEST=1 -o sqlite_tester \ + $(TESTSRC) $(TOP)/src/tclsqlite.c \ libsqlite.a $(LIBGDBM) $(LIBTCL) test: sqlite_tester sqlite diff --git a/manifest b/manifest index 829ede70ae..4192a2b17e 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Getting\sready\sto\sredo\sthe\sjournal\sfile\sformat.\s(CVS\s210) -D 2001-04-14T16:38:23 +C Pager\sis\sworking,\smostly.\s(CVS\s211) +D 2001-04-15T00:37:09 F COPYRIGHT 74a8a6531a42e124df07ab5599aad63870fa0bd4 -F Makefile.in 4775f7fd1ed543606bb24fa3fada1bc90a23a6b9 +F Makefile.in ac01d6145714b0d1c9e99382caf03cf30d6f4c8d F README 51f6a4e7408b34afa5bc1c0485f61b6a4efb6958 F VERSION 71874cb7e2a53c2bd22bb6affa7d223dd94a7a13 F configure 260d3be664b6d9b4d2d985e66b6dae1ef723c86e x @@ -9,7 +9,7 @@ F configure.in 6940e3f88bf3d28a10c73b06ab99fd3a7e039a61 F doc/lemon.html e233a3e97a779c7a87e1bc4528c664a58e49dd47 F doc/report1.txt 734cbae63b1310cc643fe5e9e3da1ab55a79b99e F src/TODO 38a68a489e56e9fd4a96263e0ff9404a47368ad4 -F src/build.c 6afbb6106c1e8c771ffcb81107b755e200310574 +F src/build.c 4f6a2d551c56342cd4a0420654835be3ad179651 F src/dbbe.c ec82c602c598748204a61a35ab0c31e34ca58223 F src/dbbe.h 7235b15c6c5d8be0c4da469cef9620cee70b1cc8 F src/dbbegdbm.c 9d3a3c18b27f9f2533a3aaa3741e8668bdda7e98 @@ -26,8 +26,8 @@ F src/ex/sizes.tcl f54bad4a2ac567624be59131a6ee42d71b41a3d7 F src/expr.c c4c24c3af1eba094a816522eb0e085bed518ee16 F src/insert.c aa528e20a787af85432a61daaea6df394bd251d7 F src/main.c 92ce30a89f622ba36cc8b7d912829e14a480722c -F src/pager.c 67227909b9a16a2463bbfddf19347fcab7a251ae -F src/pager.h 889c5cf517ad30704e295540793c893ac843fd5f +F src/pager.c 52dc932c937cfd5ccdd3c40527ea57878994a99c +F src/pager.h 8678d9a97fdf1c111b619a13a351e5c8ab97cb81 F src/parse.y 8fc096948994a7ffbf61ba13129cc589f794a9cb F src/printf.c b1e22a47be8cdf707815647239991e08e8cb69f9 F src/random.c b36c3f57dc80c8f354e6bfbf39cf1e1de021d54a @@ -37,8 +37,9 @@ F src/shell.tcl 27ecbd63dd88396ad16d81ab44f73e6c0ea9d20e F src/sqlite.h.in 3e5906f72608f0fd4394dfbb1d7e8d35b8353677 F src/sqliteInt.h fc1000f023b41882bbdb8db4f80172f77b44307b F src/table.c adcaf074f6c1075e86359174e68701fa2acfc4d6 -F src/tclsqlite.c 83dcbf015ea0319c2a97514b8b812a12d621e40a +F src/tclsqlite.c 1f2bf4691a6bd81fbff1856ae4a12db24d1265f7 F src/test1.c abb3cb427e735ae87e6533f5b3b7164b7da91bc4 +F src/test2.c 49e27f03d1866635c2c4539e0983e5590a322872 F src/tokenize.c 0118b57702cb6550769316e8443b06760b067acf F src/update.c 0cf789656a936d4356668393267692fa4b03ffc6 F src/util.c 1b396ac34e30dd6222d82e996c17b161bbc906bc @@ -58,6 +59,7 @@ F test/insert2.test 732405e30331635af8d159fccabe835eea5cd0c6 F test/lock.test bca7d53de73138b1f670a2fbdb1f481ff7eaa45a F test/main.test da635f9e078cd21ddf074e727381a715064489ff F test/malloc.test 3daa97f6a9577d8f4c6e468b274333af19ce5861 +F test/pager.test 2317ccf5821f15a637b84fa5f4d36b4c85a52a12 F test/printf.test 4c71871e1a75a2dacb673945fc13ddb30168798f F test/rowid.test 128453599def7435e988216f7fe89c7450b8a9a3 F test/select1.test 223507655cdb4f9901d83fa7f5c5328e022c211f @@ -88,7 +90,7 @@ F www/arch.fig 4f246003b7da23bd63b8b0af0618afb4ee3055c8 F www/arch.png 8dae0766d42ed3de9ed013c1341a5792bcf633e6 F www/arch.tcl a40380c1fe0080c43e6cc5c20ed70731511b06be F www/c_interface.tcl ddca19005c47dd5a15882addc02fff5de83d8ed9 -F www/changes.tcl e89e31b688833f6393dc1add45e3b4bd1da966b8 +F www/changes.tcl 822b425cc50cb8e21563dd1aa0e4b79cf780f3dc F www/crosscompile.tcl c99efacb3aefaa550c6e80d91b240f55eb9fd33e F www/dynload.tcl 02eb8273aa78cfa9070dd4501dca937fb22b466c F www/fileformat.tcl cfb7fba80b7275555281ba2f256c00734bcdd1c9 @@ -99,7 +101,7 @@ F www/opcode.tcl cb3a1abf8b7b9be9f3a228d097d6bf8b742c2b6f F www/sqlite.tcl cb0d23d8f061a80543928755ec7775da6e4f362f F www/tclsqlite.tcl 06f81c401f79a04f2c5ebfb97e7c176225c0aef2 F www/vdbe.tcl 0c8aaa529dd216ccbf7daaabd80985e413d5f9ad -P 3bde128418fe70a2fd62bf9e013999827a16053c -R e5aeeb4c78022c778bcfebd97e61f590 +P 42c2f3fe68c8a6bab96e035371ecd64296c5491f +R 558a8ad440acd3c96149b89f77d45a0d U drh -Z 3ef685e141624dcddb66699cb8550c69 +Z 6080de1bb9d6e86286ceffe08ee69ef0 diff --git a/manifest.uuid b/manifest.uuid index f175fd505f..7ab935bd25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42c2f3fe68c8a6bab96e035371ecd64296c5491f \ No newline at end of file +f82fa7070ae281804c019e6b05cd767dadaf0827 \ No newline at end of file diff --git a/src/build.c b/src/build.c index df2526960b..b3a5eff57f 100644 --- a/src/build.c +++ b/src/build.c @@ -33,7 +33,7 @@ ** COPY ** VACUUM ** -** $Id: build.c,v 1.27 2001/04/11 14:28:42 drh Exp $ +** $Id: build.c,v 1.28 2001/04/15 00:37:09 drh Exp $ */ #include "sqliteInt.h" @@ -196,7 +196,7 @@ static void sqliteDeleteIndex(sqlite *db, Index *pIndex){ ** Table. No changes are made to disk by this routine. ** ** This routine just deletes the data structure. It does not unlink -** the table data structure from the hash table. But does it destroy +** the table data structure from the hash table. But it does destroy ** memory structures of the indices associated with the table. ** ** Indices associated with the table are unlinked from the "db" @@ -236,7 +236,13 @@ char *sqliteTableNameFromToken(Token *pName){ /* ** Begin constructing a new table representation in memory. This is ** the first of several action routines that get called in response -** to a CREATE TABLE statement. +** to a CREATE TABLE statement. In particular, this routine is called +** after seeing tokens "CREATE" and "TABLE" and the table name. The +** pStart token is the CREATE and pName is the table name. +** +** The new table is constructed in files of the pParse structure. As +** more of the CREATE TABLE statement is parsed, additional action +** routines are called to build up more of the table. */ void sqliteStartTable(Parse *pParse, Token *pStart, Token *pName){ Table *pTable; @@ -273,6 +279,11 @@ void sqliteStartTable(Parse *pParse, Token *pStart, Token *pName){ /* ** Add a new column to the table currently being constructed. +** +** The parser calls this routine once for each column declaration +** in a CREATE TABLE statement. sqliteStartTable() gets called +** first to get things going. Then this routine is called for each +** column. */ void sqliteAddColumn(Parse *pParse, Token *pName){ Table *p; @@ -295,6 +306,9 @@ void sqliteAddColumn(Parse *pParse, Token *pName){ ** The given token is the default value for the last column added to ** the table currently under construction. If "minusFlag" is true, it ** means the value token was preceded by a minus sign. +** +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. */ void sqliteAddDefaultValue(Parse *pParse, Token *pVal, int minusFlag){ Table *p; @@ -403,6 +417,7 @@ Table *sqliteTableFromToken(Parse *pParse, Token *pTok){ /* ** This routine is called to do the work of a DROP TABLE statement. +** pName is the name of the table to be dropped. */ void sqliteDropTable(Parse *pParse, Token *pName){ Table *pTable; diff --git a/src/pager.c b/src/pager.c index 41650d98fa..9402b10d3c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -27,13 +27,15 @@ ** all writes in order to support rollback. Locking is used to limit ** access to one or more reader or on writer. ** -** @(#) $Id: pager.c,v 1.2 2001/04/14 16:38:23 drh Exp $ +** @(#) $Id: pager.c,v 1.3 2001/04/15 00:37:09 drh Exp $ */ +#include "sqliteInt.h" #include "pager.h" #include #include #include #include +#include /* ** The page cache as a whole is always in one of the following @@ -66,15 +68,18 @@ #define SQLITE_READLOCK 1 #define SQLITE_WRITELOCK 2 + /* ** Each in-memory image of a page begins with the following header. */ +typedef struct PgHdr PgHdr; struct PgHdr { Pager *pPager; /* The pager to which this page belongs */ Pgno pgno; /* The page number for this page */ PgHdr *pNextHash, *pPrevHash; /* Hash collision chain for PgHdr.pgno */ int nRef; /* Number of users of this page */ - PgHdr *pNext, *pPrev; /* Freelist of pages where nRef==0 */ + PgHdr *pNextFree, *pPrevFree; /* Freelist of pages where nRef==0 */ + PgHdr *pNextAll, *pPrevAll; /* A list of all pages */ char inJournal; /* TRUE if has been written to journal */ char dirty; /* TRUE if we need to write back changes */ /* SQLITE_PAGE_SIZE bytes of page data follow this header */ @@ -87,16 +92,11 @@ struct PgHdr { #define PGHDR_TO_DATA(P) ((void*)(&(P)[1])) #define DATA_TO_PGHDR(D) (&((PgHdr*)(D))[-1]) -/* -** The number of page numbers that will fit on one page. -*/ -#define SQLITE_INDEX_SIZE (SQLITE_PAGE_SIZE/sizeof(Pgno)) - /* ** How big to make the hash table used for locating in-memory pages ** by page number. */ -#define N_PG_HASH 353 +#define N_PG_HASH 101 /* ** A open page cache is an instance of the following structure. @@ -105,50 +105,152 @@ struct Pager { char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int fd, jfd; /* File descriptors for database and journal */ - int nRef; /* Sum of PgHdr.nRef */ int dbSize; /* Number of pages in the file */ int origDbSize; /* dbSize before the current change */ - int jSize; /* Number of pages in the journal */ - int nIdx; /* Number of entries in aIdx[] */ int nPage; /* Total number of in-memory pages */ + int nRef; /* Number of in-memory pages with PgHdr.nRef>0 */ int mxPage; /* Maximum number of pages to hold in cache */ - char state; /* SQLITE_UNLOCK, _READLOCK or _WRITELOCK */ - char ioErr; /* True if an I/O error has occurred */ + int nHit, nMiss, nOvfl; /* Cache hits, missing, and LRU overflows */ + unsigned char state; /* SQLITE_UNLOCK, _READLOCK or _WRITELOCK */ + unsigned char errMask; /* One of several kinds of errors */ PgHdr *pFirst, *pLast; /* List of free pages */ + PgHdr *pAll; /* List of all pages */ PgHdr *aHash[N_PG_HASH]; /* Hash table to map page number of PgHdr */ - Pgno aIdx[SQLITE_INDEX_SIZE]; /* Current journal index page */ +}; + +/* +** These are bits that can be set in Pager.errMask. +*/ +#define PAGER_ERR_FULL 0x01 /* a write() failed */ +#define PAGER_ERR_MEM 0x02 /* malloc() failed */ +#define PAGER_ERR_LOCK 0x04 /* error in the locking protocol */ +#define PAGER_ERR_CORRUPT 0x08 /* database or journal corruption */ + +/* +** The journal file contains page records in the following +** format. +*/ +typedef struct PageRecord PageRecord; +struct PageRecord { + Pgno pgno; /* The page number */ + char aData[SQLITE_PAGE_SIZE]; /* Original data for page pgno */ +}; + +/* +** Journal files begin with the following magic string. This data +** is completely random. It is used only as a sanity check. +*/ +static const unsigned char aJournalMagic[] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd4, }; /* ** Hash a page number */ -#define sqlite_pager_hash(PN) ((PN)%N_PG_HASH) +#define pager_hash(PN) ((PN)%N_PG_HASH) /* ** Attempt to acquire a read lock (if wrlock==0) or a write lock (if wrlock==1) ** on the database file. Return 0 on success and non-zero if the lock ** could not be acquired. */ -static int sqlite_pager_lock(int fd, int wrlock){ +static int pager_lock(int fd, int wrlock){ + int rc; struct flock lock; - lock.l_type = write_lock ? F_WRLCK : F_RDLCK; - return fcntl(fd, F_SETLK, &lock)!=0; + lock.l_type = wrlock ? F_WRLCK : F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + rc = fcntl(fd, F_SETLK, &lock); + return rc!=0; } /* ** Unlock the database file. */ -static int sqlite_pager_unlock(fd){ +static int pager_unlock(fd){ + int rc; struct flock lock; lock.l_type = F_UNLCK; - return fcntl(fd, F_SETLK, &lock)!=0; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + rc = fcntl(fd, F_SETLK, &lock); + return rc!=0; +} + +/* +** Move the cursor for file descriptor fd to the point whereto from +** the beginning of the file. +*/ +static int pager_seek(int fd, off_t whereto){ + lseek(fd, whereto, SEEK_SET); + return SQLITE_OK; +} + +/* +** Truncate the given file so that it contains exactly mxPg pages +** of data. +*/ +static int pager_truncate(int fd, Pgno mxPg){ + int rc; + rc = ftruncate(fd, mxPg*SQLITE_PAGE_SIZE); + return rc!=0 ? SQLITE_IOERR : SQLITE_OK; +} + +/* +** Read nBytes of data from fd into pBuf. If the data cannot be +** read or only a partial read occurs, then the unread parts of +** pBuf are filled with zeros and this routine returns SQLITE_IOERR. +** If the read is completely successful, return SQLITE_OK. +*/ +static int pager_read(int fd, void *pBuf, int nByte){ + int rc; + rc = read(fd, pBuf, nByte); + if( rc<0 ){ + memset(pBuf, 0, nByte); + return SQLITE_IOERR; + } + if( rcerrMask into an approprate +** return code. +*/ +static int pager_errcode(Pager *pPager){ + int rc = SQLITE_OK; + if( pPager->errMask & PAGER_ERR_LOCK ) rc = SQLITE_PROTOCOL; + if( pPager->errMask & PAGER_ERR_FULL ) rc = SQLITE_FULL; + if( pPager->errMask & PAGER_ERR_MEM ) rc = SQLITE_NOMEM; + if( pPager->errMask & PAGER_ERR_CORRUPT ) rc = SQLITE_CORRUPT; + return rc; } /* ** Find a page in the hash table given its page number. Return ** a pointer to the page or NULL if not found. */ -static PgHdr *sqlite_pager_lookup(Pager *pPager, Pgno pgno){ +static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ PgHdr *p = pPager->aHash[pgno % N_PG_HASH]; while( p && p->pgno!=pgno ){ p = p->pNextHash; @@ -162,21 +264,23 @@ static PgHdr *sqlite_pager_lookup(Pager *pPager, Pgno pgno){ ** opened. Any outstanding pages are invalidated and subsequent attempts ** to access those pages will likely result in a coredump. */ -static void sqlite_pager_reset(Pager *pPager){ +static void pager_reset(Pager *pPager){ PgHdr *pPg, *pNext; - for(pPg=pPager->pFirst; pPg; pPg=pNext){ - pNext = pPg->pNext; - sqlite_free(pPg); + for(pPg=pPager->pAll; pPg; pPg=pNext){ + pNext = pPg->pNextAll; + sqliteFree(pPg); } pPager->pFirst = 0; - pPager->pNext = 0; + pPager->pLast = 0; + pPager->pAll = 0; memset(pPager->aHash, 0, sizeof(pPager->aHash)); pPager->nPage = 0; if( pPager->state==SQLITE_WRITELOCK ){ - sqlite_pager_rollback(pPager); + sqlitepager_rollback(pPager); } - sqlite_pager_unlock(pPager->fd); + pager_unlock(pPager->fd); pPager->state = SQLITE_UNLOCK; + pPager->dbSize = -1; pPager->nRef = 0; } @@ -193,178 +297,155 @@ static void sqlite_pager_reset(Pager *pPager){ ** should get a write lock on the database without first getting a lock ** on the journal. So this routine should never fail. But it can fail ** if another process is not playing by the rules. If it does fail, -** all in-memory cache pages are invalidated and this routine returns -** SQLITE_PROTOCOL. SQLITE_OK is returned on success. +** all in-memory cache pages are invalidated, the PAGER_ERR_LOCK bit +** is set in pPager->errMask, and this routine returns SQLITE_PROTOCOL. +** SQLITE_OK is returned on success. */ -static int sqlite_pager_unwritelock(Pager *pPager){ +static int pager_unwritelock(Pager *pPager){ int rc; - assert( pPager->state==SQLITE_WRITELOCK ); - sqlite_pager_unlock(pPager->fd); - rc = sqlite_pager_lock(pPager->fd, 0); + PgHdr *pPg; + if( pPager->state!=SQLITE_WRITELOCK ) return SQLITE_OK; + pager_unlock(pPager->fd); + rc = pager_lock(pPager->fd, 0); unlink(pPager->zJournal); close(pPager->jfd); pPager->jfd = -1; + for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ + pPg->inJournal = 0; + pPg->dirty = 0; + } if( rc!=SQLITE_OK ){ pPager->state = SQLITE_UNLOCK; - sqlite_pager_reset(pPager); rc = SQLITE_PROTOCOL; + pPager->errMask |= PAGER_ERR_LOCK; }else{ + rc = SQLITE_OK; pPager->state = SQLITE_READLOCK; } return rc; } - /* ** Playback the journal and thus restore the database file to ** the state it was in before we started making changes. ** -** A journal consists of multiple segments. Every segment begins -** with a single page containing SQLITE_INDEX_SIZE page numbers. This -** first page is called the index. Most segments have SQLITE_INDEX_SIZE -** additional pages after the index. The N-th page after the index -** contains the contents of a page in the database file before that -** page was changed. The N-th entry in the index tells which page -** of the index file the data is for. +** The journal file format is as follows: There is an initial +** file-type string for sanity checking. Then there is a single +** Pgno number which is the number of pages in the database before +** changes were made. The database is truncated to this size. +** Next come zero or more page records which each page record +** consists of a Pgno, SQLITE_PAGE_SIZE bytes of data. ** -** The first segment of a journal is formatted slightly differently. -** The first segment contains an index but only SQLITE_INDEX_SIZE-1 -** data pages. The first page number in the index is actually the -** total number of pages in the original file. This number is used -** to truncate the original database file back to its original size. -** The second number in the index page is the page number for the -** first data page. And so forth. +** For playback, the pages have to be read from the journal in +** reverse order and put back into the original database file. ** -** We really need to playback the journal beginning at the end -** and working backwards toward the beginning. That way changes -** to the database are undone in the reverse order from the way they -** were applied. This is important if the same page is changed -** more than once. But many operating systems work more efficiently -** if data is read forward instead of backwards. So for efficiency -** we want to read the data in the forward direction. -** -** This routine starts with the last segment and works backwards -** toward the first. Within each segment, however, data is read -** in the forward direction for efficiency. Care is taken that -** only the first appearance of each page is copied over to the -** database file. If a page appears in the index more than once, -** only the first occurrance is written. A hash table is used to -** keep track of which pages have been written and which have not. +** If the file opened as the journal file is not a well-formed +** journal file (as determined by looking at the magic number +** at the beginning) then this routine returns SQLITE_PROTOCOL. +** If any other errors occur during playback, the database will +** likely be corrupted, so the PAGER_ERR_CORRUPT bit is set in +** pPager->errMask and SQLITE_CORRUPT is returned. If it all +** works, then this routine returns SQLITE_OK. */ -static int sqlite_pager_playback(Pager *pPager){ - int nSeg; /* Number of segments */ - int i, j; /* Loop counters */ - Pgno mxPg = 0; /* Size of the original file in pages */ - struct stat statbuf; /* Used to size the journal */ - Pgno aIndex[SQLITE_INDEX_SIZE]; /* The index page */ - char aBuf[SQLITE_PAGE_SIZE]; /* Page transfer buffer */ - Pgno aHash[SQLITE_INDEX_SIZE*2-1]; /* Hash table for pages read so far */ +static int pager_playback(Pager *pPager){ + int nRec; /* Number of Records */ + int i; /* Loop counter */ + Pgno mxPg = 0; /* Size of the original file in pages */ + struct stat statbuf; /* Used to size the journal */ + PgHdr *pPg; /* An existing page in the cache */ + PageRecord pgRec; + unsigned char aMagic[sizeof(aJournalMagic)]; int rc; - /* Figure out how many segments are in the journal. Remember that - ** the first segment is one page shorter than the others and that - ** the last segment may be incomplete. + /* Read the beginning of the journal and truncate the + ** database file back to its original size. */ - if( fstat(pPager->jfd; &statbuf)!=0 ){ + assert( pPager->jfd>=0 ); + pager_seek(pPager->jfd, 0); + rc = pager_read(pPager->jfd, aMagic, sizeof(aMagic)); + if( rc!=SQLITE_OK || memcmp(aMagic,aJournalMagic,sizeof(aMagic))!=0 ){ + return SQLITE_PROTOCOL; + } + rc = pager_read(pPager->jfd, &mxPg, sizeof(mxPg)); + if( rc!=SQLITE_OK ){ + return SQLITE_PROTOCOL; + } + pager_truncate(pPager->fd, mxPg); + pPager->dbSize = mxPg; + + /* Begin reading the journal beginning at the end and moving + ** toward the beginning. + */ + if( fstat(pPager->jfd, &statbuf)!=0 ){ return SQLITE_OK; } - if( statbuf.st_size <= SQLITE_INDEX_SIZE*SQLITE_PAGE_SIZE ){ - nSeg = 1; - }else{ - int nPage = statbuf.st_size/SQLITE_PAGE_SIZE; - nPage -= SQLITE_INDEX_SIZE; - nSeg = 1 + nPage/(SQLITE_INDEX_SIZE+1); - } + nRec = (statbuf.st_size - (sizeof(aMagic)+sizeof(Pgno))) / sizeof(PageRecord); /* Process segments beginning with the last and working backwards ** to the first. */ - for(i=nSeg-1; i>=0; i--){ + for(i=nRec-1; i>=0; i--){ /* Seek to the beginning of the segment */ - sqlite_pager_seekpage(pPager->jfd, - i>0 ? i*(SQLITE_INDEX_SIZE + 1) - 1 : 0, - SEEK_SET - ); + off_t ofst; + ofst = i*sizeof(PageRecord) + sizeof(aMagic) + sizeof(Pgno); + rc = pager_seek(pPager->jfd, ofst); + if( rc!=SQLITE_OK ) break; + rc = pager_read(pPager->jfd, &pgRec, sizeof(pgRec)); + if( rc!=SQLITE_OK ) break; - /* Initialize the hash table used to avoid copying duplicate pages */ - memset(aHash, 0, sizeof(aHash)); - - /* Read the index page */ - sqlite_pager_readpage(pPager->jfd, aIndex); - - /* Extract the original file size from the first index entry if this - ** is the first segment */ - if( i==0 ){ - mxPg = aIndex[0]; - aIndex[0] = 0; + /* Sanity checking on the page */ + if( pgRec.pgno>mxPg || pgRec.pgno==0 ){ + rc = SQLITE_CORRUPT; + break; } - /* Process pages of this segment in forward order + /* Playback the page. Update the in-memory copy of the page + ** at the same time, if there is one. */ - for(j=0; j=SQLITE_PAGE_SIZE-1 ) h = 0; - } - if( aHash[h]==pgno ){ - lseek(pPager->jfd, SQLITE_PAGE_SIZE, SEEK_CUR); - continue; - } - aHash[h] = pgno; - - /* Playback the page. Update the in-memory copy of the page - ** at the same time, if there is one. - */ - pPg = sqlite_pager_lookup(pPager, pgno); - if( pPg ){ - pBuf = PGHDR_TO_DATA(pPg); - }else{ - pBuf = aBuf; - } - sqlite_pager_readpage(pPager->jfd, pBuf); - sqlite_pager_seekpage(pPager->fd, pgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->fd, pBuf); - if( rc!=SQLITE_OK ) return rc; + pPg = pager_lookup(pPager, pgRec.pgno); + if( pPg ){ + memcpy(PGHDR_TO_DATA(pPg), pgRec.aData, SQLITE_PAGE_SIZE); } + rc = pager_seek(pPager->fd, (pgRec.pgno-1)*SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ) break; + rc = pager_write(pPager->fd, pgRec.aData, SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ) break; } - - /* Truncate the database back to its original size - */ - if( mxPg>0 ){ - ftrucate(pPager->fd, mxPg * SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ){ + pager_unwritelock(pPager); + pPager->errMask |= PAGER_ERR_CORRUPT; + rc = SQLITE_CORRUPT; + }else{ + rc = pager_unwritelock(pPager); } - return SQLITE_OK; + return rc; } /* ** Create a new page cache and put a pointer to the page cache in *ppPager. ** The file to be cached need not exist. The file is not opened until -** the first call to sqlite_pager_get() and is only held open until the -** last page is released using sqlite_pager_unref(). +** the first call to sqlitepager_get() and is only held open until the +** last page is released using sqlitepager_unref(). */ -int sqlite_pager_open(Pager **ppPager, const char *zFilename, int mxPage){ +int sqlitepager_open(Pager **ppPager, const char *zFilename, int mxPage){ Pager *pPager; int nameLen; int fd; - fd = open(zFilename, O_RDWR, 0644); + *ppPager = 0; + if( sqlite_malloc_failed ){ + return SQLITE_NOMEM; + } + fd = open(zFilename, O_RDWR|O_CREAT, 0644); if( fd<0 ){ return SQLITE_CANTOPEN; } nameLen = strlen(zFilename); pPager = sqliteMalloc( sizeof(*pPager) + nameLen*2 + 30 ); - if( pPager==0 ) return SQLITE_NOMEM; + if( pPager==0 ){ + close(fd); + return SQLITE_NOMEM; + } pPager->zFilename = (char*)&pPager[1]; pPager->zJournal = &pPager->zFilename[nameLen+1]; strcpy(pPager->zFilename, zFilename); @@ -377,6 +458,7 @@ int sqlite_pager_open(Pager **ppPager, const char *zFilename, int mxPage){ pPager->nPage = 0; pPager->mxPage = mxPage>10 ? mxPage : 10; pPager->state = SQLITE_UNLOCK; + pPager->errMask = 0; pPager->pFirst = 0; pPager->pLast = 0; memset(pPager->aHash, 0, sizeof(pPager->aHash)); @@ -387,9 +469,10 @@ int sqlite_pager_open(Pager **ppPager, const char *zFilename, int mxPage){ /* ** Return the total number of pages in the file opened by pPager. */ -int sqlite_pager_pagecount(Pager *pPager){ +int sqlitepager_pagecount(Pager *pPager){ int n; struct stat statbuf; + assert( pPager!=0 ); if( pPager->dbSize>=0 ){ return pPager->dbSize; } @@ -398,7 +481,7 @@ int sqlite_pager_pagecount(Pager *pPager){ }else{ n = statbuf.st_size/SQLITE_PAGE_SIZE; } - if( pPager->state!=SQLITE_NOLOCK ){ + if( pPager->state!=SQLITE_UNLOCK ){ pPager->dbSize = n; } return n; @@ -413,17 +496,16 @@ int sqlite_pager_pagecount(Pager *pPager){ ** with this page cache after this function returns will likely ** result in a coredump. */ -int sqlite_pager_close(Pager *pPager){ - int i; - PgHdr *pPg; +int sqlitepager_close(Pager *pPager){ + PgHdr *pPg, *pNext; switch( pPager->state ){ case SQLITE_WRITELOCK: { - sqlite_pager_rollback(pPager); - sqlite_pager_unlock(pPager->fd); + sqlitepager_rollback(pPager); + pager_unlock(pPager->fd); break; } case SQLITE_READLOCK: { - sqlite_pager_unlock(pPager->fd); + pager_unlock(pPager->fd); break; } default: { @@ -431,12 +513,9 @@ int sqlite_pager_close(Pager *pPager){ break; } } - for(i=0; iaHash[i]; pPg; pPg=pNext){ - pNext = pPg->pNextHash; - sqliteFree(pPg); - } + for(pPg=pPager->pAll; pPg; pPg=pNext){ + pNext = pPg->pNextAll; + sqliteFree(pPg); } if( pPager->fd>=0 ) close(pPager->fd); assert( pPager->jfd<0 ); @@ -447,25 +526,41 @@ int sqlite_pager_close(Pager *pPager){ /* ** Return the page number for the given page data */ -int sqlite_pager_pagenumber(void *pData){ +Pgno sqlitepager_pagenumber(void *pData){ PgHdr *p = DATA_TO_PGHDR(pData); return p->pgno; } /* -** Acquire a page +** Acquire a page. +** +** A read lock is obtained for the first page acquired. The lock +** is dropped when the last page is released. +** +** The acquisition might fail for several reasons. In all cases, +** an appropriate error code is returned and *ppPage is set to NULL. */ -int sqlite_pager_get(Pager *pPager, int pgno, void **ppPage){ +int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){ PgHdr *pPg; + /* Make sure we have not hit any critical errors. + */ + if( pPager==0 || pgno==0 ){ + return SQLITE_ERROR; + } + if( pPager->errMask & ~(PAGER_ERR_FULL) ){ + return pager_errcode(pPager); + } + /* If this is the first page accessed, then get a read lock ** on the database file. */ if( pPager->nRef==0 ){ - if( sqlite_pager_lock(pPager->fd, 0)!=0 ){ + if( pager_lock(pPager->fd, 0)!=0 ){ *ppPage = 0; return SQLITE_BUSY; } + pPager->state = SQLITE_READLOCK; /* If a journal file exists, try to play it back. */ @@ -476,16 +571,18 @@ int sqlite_pager_get(Pager *pPager, int pgno, void **ppPage){ ** we cannot get exclusive access to the journal file */ pPager->jfd = open(pPager->zJournal, O_RDONLY, 0); - if( pPager->jfd<0 || sqlite_pager_lock(pPager->jfd, 1)!=0 ){ + if( pPager->jfd<0 || pager_lock(pPager->jfd, 1)!=0 ){ if( pPager->jfd>=0 ){ close(pPager->jfd); pPager->jfd = -1; } - sqlite_pager_unlock(pPager->fd); + pager_unlock(pPager->fd); *ppPage = 0; return SQLITE_BUSY; } /* Get a write lock on the database */ - sqlite_pager_unlock(pPager->fd); - if( sqlite_pager_lock(pPager->fd, 1)!=0 ){ + pager_unlock(pPager->fd); + if( pager_lock(pPager->fd, 1)!=0 ){ + close(pPager->jfd); + pPager->jfd = -1; *ppPage = 0; return SQLITE_PROTOCOL; } @@ -493,36 +590,77 @@ int sqlite_pager_get(Pager *pPager, int pgno, void **ppPage){ /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. */ - sqlite_pager_playback(pPager); - rc = sqlite_pager_unwritelock(pPager); - if( rc!=SQLITE_OK ){ return SQLITE_PROTOCOL; } + rc = pager_playback(pPager); + if( rc!=SQLITE_OK ){ + return rc; + } } pPg = 0; + pPager->nMiss++; }else{ /* Search for page in cache */ - pPg = sqlite_pager_lookup(pPager, pgno); + pPg = pager_lookup(pPager, pgno); + pPager->nHit++; } if( pPg==0 ){ + /* The requested page is not in the page cache. */ int h; if( pPager->nPagemxPage || pPager->pFirst==0 ){ /* Create a new page */ - pPg = sqlite_malloc( sizeof(*pPg) + SQLITE_PAGE_SIZE ); + pPg = sqliteMalloc( sizeof(*pPg) + SQLITE_PAGE_SIZE ); + if( pPg==0 ){ + *ppPage = 0; + pager_unwritelock(pPager); + pPager->errMask |= PAGER_ERR_MEM; + return SQLITE_NOMEM; + } pPg->pPager = pPager; + pPg->pNextAll = pPager->pAll; + if( pPager->pAll ){ + pPager->pAll->pPrevAll = pPg; + } + pPg->pPrevAll = 0; + pPager->nPage++; }else{ - /* Recycle an older page */ + /* Recycle an older page. First locate the page to be recycled. + ** Try to find one that is not dirty and is near the head of + ** of the free list */ + int cnt = 4; pPg = pPager->pFirst; + while( pPg->dirty && 0pNextFree; + } + if( pPg==0 || pPg->dirty ) pPg = pPager->pFirst; + assert( pPg->nRef==0 ); + + /* If the page to be recycled is dirty, sync the journal and write + ** the old page into the database. */ if( pPg->dirty ){ int rc; - sqlite_pager_seekpage(pPager->fd, pPg->pgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->fd, PGHDR_TO_DATA(pPg)); - if( rc!=SQLITE_OK ){ + assert( pPg->inJournal==1 ); + assert( pPager->state==SQLITE_WRITELOCK ); + rc = fsync(pPager->jfd); + if( rc!=0 ){ + rc = sqlitepager_rollback(pPager); *ppPage = 0; + if( rc==SQLITE_OK ) rc = SQLITE_IOERR; return rc; } - } - pPager->pFirst = pPg->pNext; + pager_seek(pPager->fd, (pPg->pgno-1)*SQLITE_PAGE_SIZE); + rc = pager_write(pPager->fd, PGHDR_TO_DATA(pPg), SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ){ + rc = sqlitepager_rollback(pPager); + *ppPage = 0; + if( rc==SQLITE_OK ) rc = SQLITE_FULL; + return rc; + } + } + + /* Unlink the old page from the free list and the hash table + */ + pPager->pFirst = pPg->pNextFree; if( pPager->pFirst ){ - pPager->pFirst->pPrev = 0; + pPager->pFirst->pPrevFree = 0; }else{ pPager->pLast = 0; } @@ -532,36 +670,41 @@ int sqlite_pager_get(Pager *pPager, int pgno, void **ppPage){ if( pPg->pPrevHash ){ pPg->pPrevHash->pNextHash = pPg->pNextHash; }else{ - h = sqlite_pager_hash(pPg->pgno); + h = pager_hash(pPg->pgno); assert( pPager->aHash[h]==pPg ); pPager->aHash[h] = pPg->pNextHash; } + pPager->nOvfl++; } pPg->pgno = pgno; pPg->inJournal = 0; pPg->dirty = 0; pPg->nRef = 1; - h = sqlite_pager_hash(pgno); + pPager->nRef++; + h = pager_hash(pgno); pPg->pNextHash = pPager->aHash[h]; pPager->aHash[h] = pPg; if( pPg->pNextHash ){ assert( pPg->pNextHash->pPrevHash==0 ); pPg->pNextHash->pPrevHash = pPg; } - sqlite_pager_seekpage(pPager->fd, pgno, SEEK_SET); - sqlite_pager_readpage(pPager->fd, PGHDR_TO_DATA(pPg)); + pager_seek(pPager->fd, (pgno-1)*SQLITE_PAGE_SIZE); + pager_read(pPager->fd, PGHDR_TO_DATA(pPg), SQLITE_PAGE_SIZE); }else{ + /* The requested page is in the page cache. */ if( pPg->nRef==0 ){ - if( pPg->pPrev ){ - pPg->pPrev->pNext = pPg->pNext; + /* The page is currently on the freelist. Remove it. */ + if( pPg->pPrevFree ){ + pPg->pPrevFree->pNextFree = pPg->pNextFree; }else{ - pPager->pFirst = pPg->pNext; + pPager->pFirst = pPg->pNextFree; } - if( pPg->pNext ){ - pPg->pNext->pPrev = pPg->pPrev; + if( pPg->pNextFree ){ + pPg->pNextFree->pPrevFree = pPg->pPrevFree; }else{ - pPager->pLast = pPg->pPrev; + pPager->pLast = pPg->pPrevFree; } + pPager->nRef++; } pPg->nRef++; } @@ -574,31 +717,43 @@ int sqlite_pager_get(Pager *pPager, int pgno, void **ppPage){ ** ** If the number of references to the page drop to zero, then the ** page is added to the LRU list. When all references to all pages -** are released, a rollback occurs, and the lock on the database is +** are released, a rollback occurs and the lock on the database is ** removed. */ -int sqlite_pager_unref(void *pData){ +int sqlitepager_unref(void *pData){ Pager *pPager; PgHdr *pPg; + + /* Decrement the reference count for this page + */ pPg = DATA_TO_PGHDR(pData); assert( pPg->nRef>0 ); pPager = pPg->pPager; pPg->nRef--; + + /* When the number of references to a page reach 0, add the + ** page to the freelist. + */ if( pPg->nRef==0 ){ - pPg->pNext = 0; - pPg->pPrev = pPager->pLast; + pPg->pNextFree = 0; + pPg->pPrevFree = pPager->pLast; pPager->pLast = pPg; - if( pPg->pPrev ){ - pPg->pPrev->pNext = pPg; + if( pPg->pPrevFree ){ + pPg->pPrevFree->pNextFree = pPg; }else{ pPager->pFirst = pPg; } + + /* When all pages reach the freelist, drop the read lock from + ** the database file. + */ + pPager->nRef--; + assert( pPager->nRef>=0 ); + if( pPager->nRef==0 ){ + pager_reset(pPager); + } } - pPager->nRef--; - assert( pPager->nRef>=0 ); - if( pPager->nRef==0 ){ - sqlite_pager_reset(pPager); - } + return SQLITE_OK; } /* @@ -611,87 +766,120 @@ int sqlite_pager_unref(void *pData){ ** lock could not be acquired, this routine returns SQLITE_BUSY. The ** calling routine must check for that routine and be careful not to ** change any page data until this routine returns SQLITE_OK. +** +** If the journal file could not be written because the disk is full, +** then this routine returns SQLITE_FULL and does an immediate rollback. +** All subsequent write attempts also return SQLITE_FULL until there +** is a call to sqlitepager_commit() or sqlitepager_rollback() to +** reset. */ -int sqlite_pager_write(void *pData){ +int sqlitepager_write(void *pData){ PgHdr *pPg = DATA_TO_PGHDR(pData); Pager *pPager = pPg->pPager; int rc; + if( pPager->errMask ){ + return pager_errcode(pPager); + } + pPg->dirty = 1; if( pPg->inJournal ){ return SQLITE_OK; } - if( pPager->state==SQLITE_UNLOCK ){ return SQLITE_PROTOCOL; } + assert( pPager->state!=SQLITE_UNLOCK ); if( pPager->state==SQLITE_READLOCK ){ pPager->jfd = open(pPager->zJournal, O_RDWR|O_CREAT, 0644); if( pPager->jfd<0 ){ return SQLITE_CANTOPEN; } - if( sqlite_pager_lock(pPager->jfd, 1) ){ + if( pager_lock(pPager->jfd, 1) ){ close(pPager->jfd); pPager->jfd = -1; return SQLITE_BUSY; } - sqlite_pager_unlock(pPager->fd); - if( sqlite_pager_lock(pPager->fd, 1) ){ + pager_unlock(pPager->fd); + if( pager_lock(pPager->fd, 1) ){ close(pPager->jfd); pPager->jfd = -1; pPager->state = SQLITE_UNLOCK; - sqlite_pager_reset(pPager); + pPager->errMask |= PAGER_ERR_LOCK; return SQLITE_PROTOCOL; } pPager->state = SQLITE_WRITELOCK; - pPager->jSize = 1; - pPager->aIdx[0] = pPager->dbSize; + sqlitepager_pagecount(pPager); pPager->origDbSize = pPager->dbSize; - pPager->nIdx = 1; + rc = pager_write(pPager->jfd, aJournalMagic, sizeof(aJournalMagic)); + if( rc==SQLITE_OK ){ + rc = pager_write(pPager->jfd, &pPager->dbSize, sizeof(Pgno)); + } + if( rc!=SQLITE_OK ){ + rc = pager_unwritelock(pPager); + if( rc==SQLITE_OK ) rc = SQLITE_FULL; + return rc; + } } - /* Write this page to the journal */ + assert( pPager->state==SQLITE_WRITELOCK ); assert( pPager->jfd>=0 ); - if( pPg->pgno >= pPager->origDbSize ){ - sqlite_pager_seekpage(pPager->fd, pPg->pgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->fd, pData); - pPg->inJournal = 1; - return rc; + if( pPg->pgno <= pPager->origDbSize ){ + rc = pager_write(pPager->jfd, &pPg->pgno, sizeof(Pgno)); + if( rc==SQLITE_OK ){ + rc = pager_write(pPager->jfd, pData, SQLITE_PAGE_SIZE); + } + if( rc!=SQLITE_OK ){ + sqlitepager_rollback(pPager); + pPager->errMask |= PAGER_ERR_FULL; + return rc; + } } - pPager->aIdx[pPager->nIdx++] = pPg->pgno; - sqlite_pager_seekpage(pPager->jfd, pPager->jSize++, SEEK_SET); - rc = sqlite_pager_write(pPager->jfd, pData); pPg->inJournal = 1; - if( pPager->nIdx==SQLITE_INDEX_SIZE ){ - sqlite_pager_seekpage(pPager->jfd, pPager->idxPgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->jfd, &pPager->aIdx); - pPager->nIdx = 0; - pPager->jSize++; - } return rc; } /* ** Commit all changes to the database and release the write lock. +** +** If the commit fails for any reason, a rollback attempt is made +** and an error code is returned. If the commit worked, SQLITE_OK +** is returned. */ -int sqlite_pager_commit(Pager*){ +int sqlitepager_commit(Pager *pPager){ int i, rc; PgHdr *pPg; - assert( pPager->state==SQLITE_WRITELOCK ); + + if( pPager->errMask==PAGER_ERR_FULL ){ + rc = sqlitepager_rollback(pPager); + if( rc==SQLITE_OK ) rc = SQLITE_FULL; + return rc; + } + if( pPager->errMask!=0 ){ + rc = pager_errcode(pPager); + return rc; + } + if( pPager->state!=SQLITE_WRITELOCK ){ + return SQLITE_ERROR; + } assert( pPager->jfd>=0 ); - memset(&pPager->aIdx[&pPager->nIdx], 0, - (SQLITE_INDEX_SIZE - pPager->nIdx)*sizeof(Pgno)); - sqlite_pager_seekpage(pPager->jfd, pPager->idxPgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->jfd, &pPager->aIdx); if( fsync(pPager->jfd) ){ - return SQLITE_IOERR; + goto commit_abort; } for(i=0; iaHash[i]; pPg; pPg=pPg->pNextHash){ if( pPg->dirty==0 ) continue; - rc = sqlite_pager_seekpage(pPager->fd, pPg->pgno, SEEK_SET); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite_pager_writePage(pPager->fd, PGHDR_TO_DATA(pPg)); - if( rc!=SQLITE_OK ) return rc; + rc = pager_seek(pPager->fd, (pPg->pgno-1)*SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ) goto commit_abort; + rc = pager_write(pPager->fd, PGHDR_TO_DATA(pPg), SQLITE_PAGE_SIZE); + if( rc!=SQLITE_OK ) goto commit_abort; } } - if( fsync(pPager->fd) ){ - return SQLITE_IOERR; + if( fsync(pPager->fd) ) goto commit_abort; + rc = pager_unwritelock(pPager); + pPager->dbSize = -1; + return rc; + + /* Jump here if anything goes wrong during the commit process. + */ +commit_abort: + rc = sqlitepager_rollback(pPager); + if( rc==SQLITE_OK ){ + rc = SQLITE_FULL; } - rc = sqlite_pager_unwritelock(pPager); return rc; } @@ -699,17 +887,44 @@ int sqlite_pager_commit(Pager*){ ** Rollback all changes. The database falls back to read-only mode. ** All in-memory cache pages revert to their original data contents. ** The journal is deleted. +** +** This routine cannot fail unless some other process is not following +** the correct locking protocol (SQLITE_PROTOCOL) or unless some other +** process is writing trash into the journal file (SQLITE_CORRUPT) or +** unless a prior malloc() failed (SQLITE_NOMEM). Appropriate error +** codes are returned for all these occasions. Otherwise, +** SQLITE_OK is returned. */ -int sqlite_pager_rollback(Pager *pPager){ +int sqlitepager_rollback(Pager *pPager){ int rc; - if( pPager->state!=SQLITE_WRITELOCK ) return SQLITE_OK; - memset(&pPager->aIdx[&pPager->nIdx], 0, - (SQLITE_INDEX_SIZE - pPager->nIdx)*sizeof(Pgno)); - sqlite_pager_seekpage(pPager->jfd, pPager->idxPgno, SEEK_SET); - rc = sqlite_pager_writepage(pPager->jfd, &pPager->aIdx); - rc = sqlite_pager_playback(pPager); - if( rc!=SQLITE_OK ){ - rc = sqlite_pager_unwritelock(pPager); + if( pPager->errMask!=0 && pPager->errMask!=PAGER_ERR_FULL ){ + return pager_errcode(pPager); } + if( pPager->state!=SQLITE_WRITELOCK ){ + return SQLITE_OK; + } + rc = pager_playback(pPager); + if( rc!=SQLITE_OK ){ + rc = SQLITE_CORRUPT; + pPager->errMask |= PAGER_ERR_CORRUPT; + } + pPager->dbSize = -1; return rc; }; + +/* +** This routine is used for testing and analysis only. +*/ +int *sqlitepager_stats(Pager *pPager){ + static int a[9]; + a[0] = pPager->nRef; + a[1] = pPager->nPage; + a[2] = pPager->mxPage; + a[3] = pPager->dbSize; + a[4] = pPager->state; + a[5] = pPager->errMask; + a[6] = pPager->nHit; + a[7] = pPager->nMiss; + a[8] = pPager->nOvfl; + return a; +} diff --git a/src/pager.h b/src/pager.h index cec8f4ea70..8935a80e20 100644 --- a/src/pager.h +++ b/src/pager.h @@ -25,9 +25,8 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.1 2001/04/03 16:53:22 drh Exp $ +** @(#) $Id: pager.h,v 1.2 2001/04/15 00:37:09 drh Exp $ */ -#include "sqliteInt.h" /* ** The size of one page @@ -45,12 +44,14 @@ typedef unsigned int Pgno; */ typedef struct Pager Pager; -int sqlite_pager_open(Pager **ppPager, const char *zFilename); -int sqlite_pager_close(Pager *pPager); -int sqlite_pager_get(Pager *pPager, Pgno pgno, void **ppPage); -int sqlite_pager_unref(void*); -Pgno sqlite_pager_pagenumber(void*); -int sqlite_pager_write(void*); -int sqlite_pager_pagecount(Pager*); -int sqlite_pager_commit(Pager*); -int sqlite_pager_rollback(Pager*); +int sqlitepager_open(Pager **ppPager, const char *zFilename, int nPage); +int sqlitepager_close(Pager *pPager); +int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage); +int sqlitepager_unref(void*); +Pgno sqlitepager_pagenumber(void*); +int sqlitepager_write(void*); +int sqlitepager_pagecount(Pager*); +int sqlitepager_commit(Pager*); +int sqlitepager_rollback(Pager*); + +int *sqlitepager_stats(Pager*); diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 42c8eada55..6d5c2c6b0b 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -23,7 +23,7 @@ ************************************************************************* ** A TCL Interface to SQLite ** -** $Id: tclsqlite.c,v 1.17 2001/04/07 15:24:33 drh Exp $ +** $Id: tclsqlite.c,v 1.18 2001/04/15 00:37:09 drh Exp $ */ #ifndef NO_TCL /* Omit this whole file if TCL is unavailable */ @@ -507,10 +507,12 @@ int TCLSH_MAIN(int argc, char **argv){ Tcl_FindExecutable(argv[0]); interp = Tcl_CreateInterp(); Sqlite_Init(interp); -#ifdef SQLITE_TEST1 +#ifdef SQLITE_TEST { extern int Sqlitetest1_Init(Tcl_Interp*); + extern int Sqlitetest2_Init(Tcl_Interp*); Sqlitetest1_Init(interp); + Sqlitetest2_Init(interp); } #endif if( argc>=2 ){ diff --git a/src/test2.c b/src/test2.c new file mode 100644 index 0000000000..036f454e28 --- /dev/null +++ b/src/test2.c @@ -0,0 +1,384 @@ +/* +** Copyright (c) 2001 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the GNU General Public +** License as published by the Free Software Foundation; either +** version 2 of the License, or (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** General Public License for more details. +** +** You should have received a copy of the GNU General Public +** License along with this library; if not, write to the +** Free Software Foundation, Inc., 59 Temple Place - Suite 330, +** Boston, MA 02111-1307, USA. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +************************************************************************* +** Code for testing the pager.c module in SQLite. This code +** is not included in the SQLite library. It is used for automated +** testing of the SQLite library. +** +** $Id: test2.c,v 1.1 2001/04/15 00:37:09 drh Exp $ +*/ +#include "sqliteInt.h" +#include "pager.h" +#include "tcl.h" +#include +#include + +/* +** Interpret an SQLite error number +*/ +static char *errorName(int rc){ + char *zName; + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + default: zName = "SQLITE_Unknown"; break; + } + return zName; +} + +/* +** Usage: pager_open FILENAME N-PAGE +** +** Open a new pager +*/ +static int pager_open( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + int nPage; + int rc; + char zBuf[100]; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " FILENAME N-PAGE\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[2], &nPage) ) return TCL_ERROR; + rc = sqlitepager_open(&pPager, argv[1], nPage); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + sprintf(zBuf,"0x%x",(int)pPager); + Tcl_AppendResult(interp, zBuf, 0); + return TCL_OK; +} + +/* +** Usage: pager_close ID +** +** Close the given pager. +*/ +static int pager_close( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + rc = sqlitepager_close(pPager); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** Usage: pager_rollback ID +** +** Rollback changes +*/ +static int pager_rollback( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + rc = sqlitepager_rollback(pPager); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** Usage: pager_commit ID +** +** Commit all changes +*/ +static int pager_commit( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + rc = sqlitepager_commit(pPager); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** Usage: pager_stats ID +** +** Return pager statistics. +*/ +static int pager_stats( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + int i, *a; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + a = sqlitepager_stats(pPager); + for(i=0; i<9; i++){ + static char *zName[] = { + "ref", "page", "max", "size", "state", "err", + "hit", "miss", "ovfl", + }; + char zBuf[100]; + Tcl_AppendElement(interp, zName[i]); + sprintf(zBuf,"%d",a[i]); + Tcl_AppendElement(interp, zBuf); + } + return TCL_OK; +} + +/* +** Usage: pager_pagecount ID +** +** Return the size of the database file. +*/ +static int pager_pagecount( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + char zBuf[100]; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + sprintf(zBuf,"%d",sqlitepager_pagecount(pPager)); + Tcl_AppendResult(interp, zBuf, 0); + return TCL_OK; +} + +/* +** Usage: page_get ID PGNO +** +** Return a pointer to a page from the database. +*/ +static int page_get( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + Pager *pPager; + char zBuf[100]; + void *pPage; + int pgno; + int rc; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ID PGNO\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR; + if( Tcl_GetInt(interp, argv[2], &pgno) ) return TCL_ERROR; + rc = sqlitepager_get(pPager, pgno, &pPage); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + sprintf(zBuf,"0x%x",(int)pPage); + Tcl_AppendResult(interp, zBuf, 0); + return TCL_OK; +} + +/* +** Usage: page_unref PAGE +** +** Drop a pointer to a page. +*/ +static int page_unref( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + void *pPage; + int rc; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " PAGE\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPage) ) return TCL_ERROR; + rc = sqlitepager_unref(pPage); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** Usage: page_read PAGE +** +** Return the content of a page +*/ +static int page_read( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + char zBuf[100]; + void *pPage; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " PAGE\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPage) ) return TCL_ERROR; + memcpy(zBuf, pPage, sizeof(zBuf)); + Tcl_AppendResult(interp, zBuf, 0); + return TCL_OK; +} + +/* +** Usage: page_number PAGE +** +** Return the page number for a page. +*/ +static int page_number( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + char zBuf[100]; + void *pPage; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " PAGE\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPage) ) return TCL_ERROR; + sprintf(zBuf, "%d", sqlitepager_pagenumber(pPage)); + Tcl_AppendResult(interp, zBuf, 0); + return TCL_OK; +} + +/* +** Usage: page_write PAGE DATA +** +** Write something into a page. +*/ +static int page_write( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + void *pPage; + int rc; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " PAGE DATA\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], (int*)&pPage) ) return TCL_ERROR; + rc = sqlitepager_write(pPage); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, errorName(rc), 0); + return TCL_ERROR; + } + strncpy((char*)pPage, argv[2], SQLITE_PAGE_SIZE-1); + ((char*)pPage)[SQLITE_PAGE_SIZE-1] = 0; + return TCL_OK; +} + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetest2_Init(Tcl_Interp *interp){ + Tcl_CreateCommand(interp, "pager_open", pager_open, 0, 0); + Tcl_CreateCommand(interp, "pager_close", pager_close, 0, 0); + Tcl_CreateCommand(interp, "pager_commit", pager_commit, 0, 0); + Tcl_CreateCommand(interp, "pager_rollback", pager_rollback, 0, 0); + Tcl_CreateCommand(interp, "pager_stats", pager_stats, 0, 0); + Tcl_CreateCommand(interp, "pager_pagecount", pager_pagecount, 0, 0); + Tcl_CreateCommand(interp, "page_get", page_get, 0, 0); + Tcl_CreateCommand(interp, "page_unref", page_unref, 0, 0); + Tcl_CreateCommand(interp, "page_read", page_read, 0, 0); + Tcl_CreateCommand(interp, "page_write", page_write, 0, 0); + Tcl_CreateCommand(interp, "page_number", page_number, 0, 0); + return TCL_OK; +} diff --git a/test/pager.test b/test/pager.test new file mode 100644 index 0000000000..cd83573505 --- /dev/null +++ b/test/pager.test @@ -0,0 +1,176 @@ +# Copyright (c) 1999, 2000 D. Richard Hipp +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Author contact information: +# drh@hwaci.com +# http://www.hwaci.com/drh/ +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is page cache subsystem. +# +# $Id: pager.test,v 1.1 2001/04/15 00:37:21 drh Exp $ + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +if {$dbprefix!="mem:" && [info commands pager_open]!=""} { + +# Basic sanity check. Open and close a pager. +# +do_test pager-1.0 { + catch {file delete -force ptf1.db} + catch {file delete -force ptf1.db-journal} + set v [catch { + set ::p1 [pager_open ptf1.db 10] + } msg] +} {0} +do_test pager-1.1 { + pager_stats $::p1 +} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 0 ovfl 0} +do_test pager-1.2 { + pager_pagecount $::p1 +} {0} +do_test pager-1.3 { + pager_stats $::p1 +} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 0 ovfl 0} +do_test pager-1.4 { + pager_close $::p1 +} {} + +# Try to write a few pages. +# +do_test pager-2.1 { + set v [catch { + set ::p1 [pager_open ptf1.db 10] + } msg] +} {0} +do_test pager-2.2 { + set v [catch { + set ::g1 [page_get $::p1 0] + } msg] + lappend v $msg +} {1 SQLITE_ERROR} +do_test pager-2.3 { + set v [catch { + set ::g1 [page_get $::p1 1] + } msg] + if {$v} {lappend v $msg} + set v +} {0} +do_test pager-2.4 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size -1 state 1 err 0 hit 0 miss 1 ovfl 0} +do_test pager-2.5 { + pager_pagecount $::p1 +} {0} +do_test pager-2.6 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} +do_test pager-2.7 { + page_number $::g1 +} {1} +do_test pager-2.8 { + page_read $::g1 +} {} +do_test pager-2.9 { + page_unref $::g1 +} {} +do_test pager-2.10 { + pager_stats $::p1 +} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 1 ovfl 0} +do_test pager-2.11 { + set ::g1 [page_get $::p1 1] + expr {$::g1!=0} +} {1} +do_test pager-2.12 { + page_number $::g1 +} {1} +do_test pager-2.13 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size -1 state 1 err 0 hit 0 miss 2 ovfl 0} +do_test pager-2.14 { + set v [catch { + page_write $::g1 "Page-One" + } msg] + lappend v $msg +} {0 {}} +do_test pager-2.15 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size 0 state 2 err 0 hit 0 miss 2 ovfl 0} +do_test pager-2.16 { + page_read $::g1 +} {Page-One} +do_test pager-2.17 { + set v [catch { + pager_commit $::p1 + } msg] + lappend v $msg +} {0 {}} +do_test pager-2.20 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size -1 state 1 err 0 hit 0 miss 2 ovfl 0} +do_test pager-2.19 { + pager_pagecount $::p1 +} {1} +do_test pager-2.21 { + pager_stats $::p1 +} {ref 1 page 1 max 10 size 1 state 1 err 0 hit 0 miss 2 ovfl 0} +do_test pager-2.22 { + page_unref $::g1 +} {} +do_test pager-2.23 { + pager_stats $::p1 +} {ref 0 page 0 max 10 size -1 state 0 err 0 hit 0 miss 2 ovfl 0} +do_test pager-2.24 { + set v [catch { + page_get $::p1 1 + } ::g1] + if {$v} {lappend v $::g1} + set v +} {0} +do_test pager-2.25 { + page_read $::g1 +} {Page-One} +do_test pager-2.26 { + set v [catch { + page_write $::g1 {page-one} + } msg] + lappend v $msg +} {0 {}} +do_test pager-2.27 { + page_read $::g1 +} {page-one} +do_test pager-2.28 { + set v [catch { + pager_rollback $::p1 + } msg] + lappend v $msg +} {0 {}} +do_test pager-2.29 { + page_read $::g1 +} {Page-One} + +do_test pager-2.99 { + pager_close $::p1 +} {} + + +} ;# end if( not mem: and has pager_open command ); + +finish_test diff --git a/www/changes.tcl b/www/changes.tcl index f5de4f6236..80e34b4e97 100644 --- a/www/changes.tcl +++ b/www/changes.tcl @@ -17,7 +17,8 @@ proc chng {date desc} { puts "

    $desc

" } -chng {2001 Apr 12 (1.0.31)} { +chng {2001 Apr 14 (1.0.31)} { +
  • Pager subsystem added but not yet used.
  • More robust handling of out-of-memory errors.
  • New tests added to the test suite.
    • }