Checkpoint code added to the pager. Regression tests work but the new APIs

have not been tested yet. (CVS 361)

FossilOrigin-Name: aaa53e113ef849e34883ead8ae584c722ad967db

manifest

@@ -1,5 +1,5 @@
-C Change\sto\sfive\sconflict\sresolution\salgorithms:\sROLLBACK,\sABORT,\sFAIL,\nIGNORE,\sand\sREPLACE.\s\sThis\scheckin\sis\scode\sonly.\s\sDocumentation\sand\ntests\sare\sstill\sneeded.\s\sAlso,\sABORT\sis\snot\sfully\simplemented.\s(CVS\s360)
+C Checkpoint\scode\sadded\sto\sthe\spager.\s\sRegression\stests\swork\sbut\sthe\snew\sAPIs\nhave\snot\sbeen\stested\syet.\s(CVS\s361)
-D 2002-01-31T15:54:21
+D 2002-02-02T15:01:16
 F Makefile.in 9fa4277413bf1d9cf91365f07d4108d7d87ed2af
 F Makefile.template 3372d45f8853afdb70bd30cc6fb50a3cd9069834
 F README a4c0ba11354ef6ba0776b400d057c59da47a4cc0
@@ -16,7 +16,7 @@ F doc/report1.txt a031aaf37b185e4fa540223cb516d3bccec7eeac
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F libtool c56e618713c9510a103bda6b95f3ea3900dcacd6
 F ltmain.sh e9ed72eb1d690f447c13945eaf69e28af531eda1
-F publish.sh 60adffbe50226a1d7d1a2930e8b7eb31535c4fe4
+F publish.sh 5b59f4aff037aafa0e4a3b6fa599495dbd73f360
 F sqlite.1 2e2bb0529ef468ade9e4322bd609d0695fb9ded9
 F src/TODO af7f3cab0228e34149cf98e073aa83d45878e7e6
 F src/btree.c c796e387da340cb628dc1e41f684fc20253f561e
@@ -27,13 +27,13 @@ F src/expr.c a2a87dbd411a508ff89dffa90505ad42dac2f920
 F src/hash.c 8f7c740ef2eaaa8decfa8751f2be30680b123e46
 F src/hash.h a5f5b3ce2d086a172c5879b0b06a27a82eac9fac
 F src/insert.c 42e89cb227ce744802622886db3572f78e72093f
-F src/main.c 637582b8b80a85b0308ca5bab8f2b42ebb002af8
+F src/main.c 300320ba68d3e5b22c2c5b2c07fa884878202181
 F src/md5.c 52f677bfc590e09f71d07d7e327bd59da738d07c
-F src/os.c c615faa4d23e742e0650e0751a6ad2a18438ad53
+F src/os.c 1953080d14098cd45e5bde88941567688efb72b1
-F src/os.h 5405a5695bf16889d4fc6caf9d42043caa41c269
+F src/os.h a17596ecc7f38a228b83ecdb661fb03ce44726d6
-F src/pager.c 1e80a3ba731e454df6bd2e58d32eeba7dd65121b
+F src/pager.c f7274d47d8c8a38ce363bfc49f8ccf9d9842e951
-F src/pager.h f78d064c780855ff70beacbeba0e2324471b26fe
+F src/pager.h b28f004e2f5541dc60cc32db01bf80cf4d056283
-F src/parse.y 90e9fc913c60b26217f4210d48a4c5c4e78f16f2
+F src/parse.y 88856227ae8472d0f4ae8514bc9561a6ca060690
 F src/printf.c 300a90554345751f26e1fc0c0333b90a66110a1d
 F src/random.c f6b36bec5ebd3edb3440224bf5bf811fe4ac9a1b
 F src/select.c fc11d5a8c2bae1b62d8028ffb111c773ad6bf161
@@ -44,7 +44,7 @@ F src/sqliteInt.h 70fd20107f4953312e76a9630a704c9405161040
 F src/table.c c89698bd5bb4b8d14722d6ee7e9be014c383d24a
 F src/tclsqlite.c b9cf346e95291cb4c4f1bf5ac1d77db6b8ad023d
 F src/test1.c 33efd350dca27c52c58c553c04fd3a6a51f13c1f
-F src/test2.c e9f99aa5ee73872819259d6612c11e55e1644321
+F src/test2.c d410dbd8a90faa466c3ab694fa0aa57f5a773aa6
 F src/test3.c d6775f95fd91f5b3cf0e2382a28e5aaeb68f745b
 F src/tokenize.c 01a09db6adf933e941db1b781789a0c175be6504
 F src/update.c 3fb7c1601bbd379e39881d6b731d3223b822188a
@@ -107,7 +107,7 @@ F www/arch.fig d5f9752a4dbf242e9cfffffd3f5762b6c63b3bcf
 F www/arch.png 82ef36db1143828a7abc88b1e308a5f55d4336f4
 F www/arch.tcl 72a0c80e9054cc7025a50928d28d9c75c02c2b8b
 F www/c_interface.tcl 82a026b1681757f13b3f62e035f3a31407c1d353
-F www/changes.tcl 3770ded78faa7a634b55fbf5316caa4cb150e5f9
+F www/changes.tcl 5c3b5b80d7144d46f100f333d4cab6184828a6c2
 F www/conflict.tcl 3f70c01680b8d763bf3305eb67f6d85fdf83b497
 F www/crosscompile.tcl 3622ebbe518927a3854a12de51344673eb2dd060
 F www/download.tcl a6d75b8b117cd33dcb090bef7e80d7556d28ebe0
@@ -122,7 +122,7 @@ F www/speed.tcl 83457b2bf6bb430900bd48ca3dd98264d9a916a5
 F www/sqlite.tcl 8b5884354cb615049aed83039f8dfe1552a44279
 F www/tclsqlite.tcl 829b393d1ab187fd7a5e978631b3429318885c49
 F www/vdbe.tcl 2013852c27a02a091d39a766bc87cff329f21218
-P cf1538d71c9ce12d5e59f367e03642cbcaf6b717
+P d0e7cf4a83e6abad7129bed356b7492dddaff474
-R b0c9c7a017538a09b4eca555b386a82f
+R 819c88d7bb6a0c5001739d080373471b
 U drh
-Z b508e059fe2aafe83b423b2a1e04f6ed
+Z 5798e92bdcf7b705413adfa7c6a6e6fe

manifest.uuid

@@ -1 +1 @@
-d0e7cf4a83e6abad7129bed356b7492dddaff474
+aaa53e113ef849e34883ead8ae584c722ad967db

publish.sh

@@ -96,6 +96,9 @@ sqlite_exec_printf
 sqlite_exec_vprintf
 sqlite_get_table_printf
 sqlite_get_table_vprintf
+sqlite_freemem
+sqlite_libversion
+sqlite_libencoding
 sqliteMalloc
 sqliteFree
 sqliteRealloc

src/main.c

@@ -14,7 +14,7 @@
 ** other files are for internal use by SQLite and should not be
 ** accessed by users of the library.
 **
-** $Id: main.c,v 1.57 2002/01/31 15:54:22 drh Exp $
+** $Id: main.c,v 1.58 2002/02/02 15:01:16 drh Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -549,3 +549,22 @@ void sqlite_busy_timeout(sqlite *db, int ms){
 void sqlite_interrupt(sqlite *db){
   db->flags |= SQLITE_Interrupt;
 }
+
+/*
+** Windows systems should call this routine to free memory that
+** is returned in the in the errmsg parameter of sqlite_open() when
+** SQLite is a DLL.  For some reason, it does not work to call free()
+** directly.
+**
+** Note that we need to call free() not sqliteFree() here, since every
+** string that is exported from SQLite should have already passed through
+** sqliteStrRealloc().
+*/
+void sqlite_freemem(void *p){ free(p); }
+
+/*
+** Windows systems need functions to call to return the sqlite_version
+** and sqlite_encoding strings.
+*/
+const char *sqlite_libversion(void){ return sqlite_version; }
+const char *sqlite_libencoding(void){ return sqlite_encoding; }

src/os.c

@@ -306,11 +306,14 @@ int sqliteOsOpenReadWrite(
 ** previously existed.  Nor do we allow the file to be a symbolic
 ** link.
 **
+** If delFlag is true, then make arrangements to automatically delete
+** the file when it is closed.
+**
 ** On success, write the file handle into *id and return SQLITE_OK.
 **
 ** On failure, return SQLITE_CANTOPEN.
 */
-int sqliteOsOpenExclusive(const char *zFilename, OsFile *id){
+int sqliteOsOpenExclusive(const char *zFilename, OsFile *id, int delFlag){
 #if OS_UNIX
   if( access(zFilename, 0)==0 ){
     return SQLITE_CANTOPEN;
@@ -331,15 +334,26 @@ int sqliteOsOpenExclusive(const char *zFilename, OsFile *id){
     return SQLITE_NOMEM;
   }
   id->locked = 0;
+  if( delFlag ){
+    unlink(zFilename);
+  }
   return SQLITE_OK;
 #endif
 #if OS_WIN
-  HANDLE h = CreateFile(zFilename,
+  HANDLE h;
+  int fileflags;
+  if( delFlag ){
+    fileflags = FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_RANDOM_ACCESS 
+                     | FILE_FLAG_DELETE_ON_CLOSE;
+  }else{
+    fileflags = FILE_FLAG_RANDOM_ACCESS;
+  }
+  h = CreateFile(zFilename,
      GENERIC_READ | GENERIC_WRITE,
      0,
      NULL,
      CREATE_ALWAYS,
-     FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS,
+     fileflags,
      NULL
   );
   if( h==INVALID_HANDLE_VALUE ){

src/os.h

@@ -58,7 +58,7 @@
 int sqliteOsDelete(const char*);
 int sqliteOsFileExists(const char*);
 int sqliteOsOpenReadWrite(const char*, OsFile*, int*);
-int sqliteOsOpenExclusive(const char*, OsFile*);
+int sqliteOsOpenExclusive(const char*, OsFile*, int);
 int sqliteOsOpenReadOnly(const char*, OsFile*);
 int sqliteOsTempFileName(char*);
 int sqliteOsClose(OsFile*);

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.36 2002/01/14 09:28:20 drh Exp $
+** @(#) $Id: pager.c,v 1.37 2002/02/02 15:01:16 drh Exp $
 */
 #include "sqliteInt.h"
 #include "pager.h"
@@ -45,6 +45,10 @@
 **                       threads can be reading or writing while one
 **                       process is writing.
 **
+**   SQLITE_CHECKPOINT   The page cache is writing to the database and
+**                       preserving its changes so that it can back them
+**                       out later if need be.
+**
 ** The page cache comes up in SQLITE_UNLOCK.  The first time a
 ** sqlite_page_get() occurs, the state transitions to SQLITE_READLOCK.
 ** After all pages have been released using sqlite_page_unref(),
@@ -55,10 +59,20 @@
 ** be in SQLITE_READLOCK before it transitions to SQLITE_WRITELOCK.)
 ** The sqlite_page_rollback() and sqlite_page_commit() functions 
 ** transition the state from SQLITE_WRITELOCK back to SQLITE_READLOCK.
+**
+** The sqlite_ckpt_begin() function moves the state from SQLITE_WRITELOCK
+** to SQLITE_CHECKPOINT.  The state transitions back to SQLITE_WRITELOCK
+** on calls to sqlite_ckpt_commit() or sqlite_ckpt_rollback().  While
+** in SQLITE_CHECKPOINT, calls to sqlite_commit() or sqlite_rollback()
+** transition directly back to SQLITE_READLOCK.
+**
+** The code does unequality comparisons on these constants so the order
+** must be preserved.
 */
 #define SQLITE_UNLOCK      0
 #define SQLITE_READLOCK    1
 #define SQLITE_WRITELOCK   2
+#define SQLITE_CHECKPOINT  3
 
 
 /*
@@ -75,6 +89,7 @@ struct PgHdr {
   PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
   PgHdr *pNextAll, *pPrevAll;    /* A list of all pages */
   char inJournal;                /* TRUE if has been written to journal */
+  char inCkpt;                   /* TRUE if written to the checkpoint journal */
   char dirty;                    /* TRUE if we need to write back changes */
   /* SQLITE_PAGE_SIZE bytes of page data follow this header */
   /* Pager.nExtra bytes of local data follow the page data */
@@ -101,9 +116,12 @@ struct Pager {
   char *zFilename;            /* Name of the database file */
   char *zJournal;             /* Name of the journal file */
   OsFile fd, jfd;             /* File descriptors for database and journal */
+  OsFile cpfd;                /* File descriptor for the checkpoint journal */
   int journalOpen;            /* True if journal file descriptors is valid */
+  int ckptOpen;               /* True if the checkpoint journal is open */
   int dbSize;                 /* Number of pages in the file */
   int origDbSize;             /* dbSize before the current change */
+  int ckptSize, ckptJSize;    /* Size of database and journal at ckpt_begin() */
   int nExtra;                 /* Add this many bytes to each in-memory page */
   void (*xDestructor)(void*); /* Call this routine when freeing pages */
   int nPage;                  /* Total number of in-memory pages */
@@ -116,6 +134,7 @@ struct Pager {
   unsigned char readOnly;     /* True for a read-only database */
   unsigned char needSync;     /* True if an fsync() is needed on the journal */
   unsigned char *aInJournal;  /* One bit for each page in the database file */
+  unsigned char *aInCkpt;     /* One bit for each page in the database */
   PgHdr *pFirst, *pLast;      /* List of free pages */
   PgHdr *pAll;                /* List of all pages */
   PgHdr *aHash[N_PG_HASH];    /* Hash table to map page number of PgHdr */
@@ -215,7 +234,7 @@ static void pager_reset(Pager *pPager){
   pPager->pAll = 0;
   memset(pPager->aHash, 0, sizeof(pPager->aHash));
   pPager->nPage = 0;
-  if( pPager->state==SQLITE_WRITELOCK ){
+  if( pPager->state>=SQLITE_WRITELOCK ){
     sqlitepager_rollback(pPager);
   }
   sqliteOsUnlock(&pPager->fd);
@@ -234,12 +253,13 @@ static void pager_reset(Pager *pPager){
 static int pager_unwritelock(Pager *pPager){
   int rc;
   PgHdr *pPg;
-  if( pPager->state!=SQLITE_WRITELOCK ) return SQLITE_OK;
+  if( pPager->state<SQLITE_WRITELOCK ) return SQLITE_OK;
   sqliteOsClose(&pPager->jfd);
   pPager->journalOpen = 0;
   sqliteOsDelete(pPager->zJournal);
   rc = sqliteOsReadLock(&pPager->fd);
   assert( rc==SQLITE_OK );
+  sqliteFree( pPager->aInCkpt );
   sqliteFree( pPager->aInJournal );
   pPager->aInJournal = 0;
   for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
@@ -250,6 +270,36 @@ static int pager_unwritelock(Pager *pPager){
   return rc;
 }
 
+/*
+** Read a single page from the journal file opened on file descriptor
+** jfd.  Playback this one page.
+*/
+static int pager_playback_one_page(Pager *pPager, OsFile *jfd){
+  int rc;
+  PgHdr *pPg;              /* An existing page in the cache */
+  PageRecord pgRec;
+
+  rc = sqliteOsRead(&pPager->jfd, &pgRec, sizeof(pgRec));
+  if( rc!=SQLITE_OK ) return rc;
+
+  /* Sanity checking on the page */
+  if( pgRec.pgno>pPager->dbSize || pgRec.pgno==0 ) return SQLITE_CORRUPT;
+
+  /* Playback the page.  Update the in-memory copy of the page
+  ** at the same time, if there is one.
+  */
+  pPg = pager_lookup(pPager, pgRec.pgno);
+  if( pPg ){
+    memcpy(PGHDR_TO_DATA(pPg), pgRec.aData, SQLITE_PAGE_SIZE);
+    memset(PGHDR_TO_EXTRA(pPg), 0, pPager->nExtra);
+  }
+  rc = sqliteOsSeek(&pPager->fd, (pgRec.pgno-1)*SQLITE_PAGE_SIZE);
+  if( rc==SQLITE_OK ){
+    rc = sqliteOsWrite(&pPager->fd, pgRec.aData, SQLITE_PAGE_SIZE);
+  }
+  return rc;
+}
+
 /*
 ** Playback the journal and thus restore the database file to
 ** the state it was in before we started making changes.  
@@ -262,16 +312,6 @@ static int pager_unwritelock(Pager *pPager){
 ** consists of a Pgno and SQLITE_PAGE_SIZE bytes of data.  See
 ** the PageRecord structure for details.
 **
-** For playback, the pages are read from the journal in
-** reverse order and put back into the original database file.
-** It used to be required to replay pages in reverse order because
-** there was a possibility of a page appearing in the journal more
-** than once.  In that case, the original value of the page was
-** the first entry so it should be reset last.  But now, a bitmap
-** is used to record every page that is in the journal.  No pages
-** are ever repeated. So we could, in theory, playback the journal
-** in the forward direction and it would still work.
-**
 ** If the file opened as the journal file is not a well-formed
 ** journal file (as determined by looking at the magic number
 ** at the beginning) then this routine returns SQLITE_PROTOCOL.
@@ -284,8 +324,6 @@ static int pager_playback(Pager *pPager){
   int nRec;                /* Number of Records */
   int i;                   /* Loop counter */
   Pgno mxPg = 0;           /* Size of the original file in pages */
-  PgHdr *pPg;              /* An existing page in the cache */
-  PageRecord pgRec;
   unsigned char aMagic[sizeof(aJournalMagic)];
   int rc;
 
@@ -321,35 +359,10 @@ static int pager_playback(Pager *pPager){
   }
   pPager->dbSize = mxPg;
   
-  /* Process segments beginning with the last and working backwards
+  /* Copy original pages out of the journal and back into the database file.
-  ** to the first.
   */
   for(i=nRec-1; i>=0; i--){
-    /* Seek to the beginning of the segment */
+    rc = pager_playback_one_page(pPager, &pPager->jfd);
-    int ofst;
-    ofst = i*sizeof(PageRecord) + sizeof(aMagic) + sizeof(Pgno);
-    rc = sqliteOsSeek(&pPager->jfd, ofst);
-    if( rc!=SQLITE_OK ) break;
-    rc = sqliteOsRead(&pPager->jfd, &pgRec, sizeof(pgRec));
-    if( rc!=SQLITE_OK ) break;
-
-    /* Sanity checking on the page */
-    if( pgRec.pgno>mxPg || pgRec.pgno==0 ){
-      rc = SQLITE_CORRUPT;
-      break;
-    }
-
-    /* Playback the page.  Update the in-memory copy of the page
-    ** at the same time, if there is one.
-    */
-    pPg = pager_lookup(pPager, pgRec.pgno);
-    if( pPg ){
-      memcpy(PGHDR_TO_DATA(pPg), pgRec.aData, SQLITE_PAGE_SIZE);
-      memset(PGHDR_TO_EXTRA(pPg), 0, pPager->nExtra);
-    }
-    rc = sqliteOsSeek(&pPager->fd, (pgRec.pgno-1)*SQLITE_PAGE_SIZE);
-    if( rc!=SQLITE_OK ) break;
-    rc = sqliteOsWrite(&pPager->fd, pgRec.aData, SQLITE_PAGE_SIZE);
     if( rc!=SQLITE_OK ) break;
   }
 
@@ -364,6 +377,78 @@ end_playback:
   return rc;
 }
 
+/*
+** Playback the checkpoint journal.
+**
+** This is similar to playing back the transaction journal but with
+** a few extra twists.
+**
+**    (1)  The original size of the database file is stored in
+**         pPager->ckptSize, not in the journal file itself.
+**
+**    (2)  In addition to playing back the checkpoint journal, also
+**         playback all pages of the transaction journal beginning
+**         at offset pPager->ckptJSize.
+*/
+static int pager_ckpt_playback(Pager *pPager){
+  int nRec;                /* Number of Records */
+  int i;                   /* Loop counter */
+  int rc;
+
+  /* Truncate the database back to its original size.
+  */
+  rc = sqliteOsTruncate(&pPager->fd, pPager->ckptSize);
+  pPager->dbSize = pPager->ckptSize;
+
+  /* Figure out how many records are in the checkpoint journal.
+  */
+  assert( pPager->ckptOpen && pPager->journalOpen );
+  sqliteOsSeek(&pPager->cpfd, 0);
+  rc = sqliteOsFileSize(&pPager->cpfd, &nRec);
+  if( rc!=SQLITE_OK ){
+    goto end_ckpt_playback;
+  }
+  nRec /= sizeof(PageRecord);
+  
+  /* Copy original pages out of the checkpoint journal and back into the
+  ** database file.
+  */
+  for(i=nRec-1; i>=0; i--){
+    rc = pager_playback_one_page(pPager, &pPager->cpfd);
+    if( rc!=SQLITE_OK ) goto end_ckpt_playback;
+  }
+
+  /* Figure out how many pages need to be copied out of the transaction
+  ** journal.
+  */
+  rc = sqliteOsSeek(&pPager->jfd, pPager->ckptJSize);
+  if( rc!=SQLITE_OK ){
+    goto end_ckpt_playback;
+  }
+  rc = sqliteOsFileSize(&pPager->jfd, &nRec);
+  if( rc!=SQLITE_OK ){
+    goto end_ckpt_playback;
+  }
+  nRec = (nRec - pPager->ckptJSize)/sizeof(PageRecord);
+  for(i=nRec-1; i>=0; i--){
+    rc = pager_playback_one_page(pPager, &pPager->jfd);
+    if( rc!=SQLITE_OK ) goto end_ckpt_playback;
+  }
+  
+
+end_ckpt_playback:
+  sqliteOsClose(&pPager->cpfd);
+  pPager->ckptOpen = 0;
+  if( rc!=SQLITE_OK ){
+    pager_unwritelock(pPager);
+    pPager->errMask |= PAGER_ERR_CORRUPT;
+    rc = SQLITE_CORRUPT;
+  }else{
+    rc = pager_unwritelock(pPager);
+  }
+  return rc;
+}
+
 /*
 ** Change the maximum number of in-memory pages that are allowed.
 */
@@ -373,6 +458,26 @@ void sqlitepager_set_cachesize(Pager *pPager, int mxPage){
   }
 }
 
+/*
+** Open a temporary file.  Write the name of the file into zName
+** (zName must be at least SQLITE_TEMPNAME_SIZE bytes long.)  Write
+** the file descriptor into *fd.  Return SQLITE_OK on success or some
+** other error code if we fail.
+**
+** The OS will automatically delete the temporary file when it is
+** closed.
+*/
+static int sqlitepager_opentemp(char *zFile, OsFile *fd){
+  int cnt = 8;
+  int rc;
+  do{
+    cnt--;
+    sqliteOsTempFileName(zFile);
+    rc = sqliteOsOpenExclusive(zFile, fd, 1);
+  }while( cnt>0 && rc!=SQLITE_OK );
+  return rc;
+}
+
 /*
 ** Create a new page cache and put a pointer to the page cache in *ppPager.
 ** The file to be cached need not exist.  The file is not locked until
@@ -405,13 +510,7 @@ int sqlitepager_open(
     rc = sqliteOsOpenReadWrite(zFilename, &fd, &readOnly);
     tempFile = 0;
   }else{
-    int cnt = 8;
+    rc = sqlitepager_opentemp(zTemp, &fd);
-    sqliteOsTempFileName(zTemp);
-    do{
-      cnt--;
-      sqliteOsTempFileName(zTemp);
-      rc = sqliteOsOpenExclusive(zTemp, &fd);
-    }while( cnt>0 && rc!=SQLITE_OK );
     zFilename = zTemp;
     tempFile = 1;
   }
@@ -431,8 +530,11 @@ int sqlitepager_open(
   strcpy(&pPager->zJournal[nameLen], "-journal");
   pPager->fd = fd;
   pPager->journalOpen = 0;
+  pPager->ckptOpen = 0;
   pPager->nRef = 0;
   pPager->dbSize = -1;
+  pPager->ckptSize = 0;
+  pPager->ckptJSize = 0;
   pPager->nPage = 0;
   pPager->mxPage = mxPage>5 ? mxPage : 10;
   pPager->state = SQLITE_UNLOCK;
@@ -493,6 +595,7 @@ int sqlitepager_pagecount(Pager *pPager){
 int sqlitepager_close(Pager *pPager){
   PgHdr *pPg, *pNext;
   switch( pPager->state ){
+    case SQLITE_CHECKPOINT:
     case SQLITE_WRITELOCK: {
       sqlitepager_rollback(pPager);
       sqliteOsUnlock(&pPager->fd);
@@ -515,7 +618,7 @@ int sqlitepager_close(Pager *pPager){
   sqliteOsClose(&pPager->fd);
   assert( pPager->journalOpen==0 );
   if( pPager->tempFile ){
-    sqliteOsDelete(pPager->zFilename);
+    /* sqliteOsDelete(pPager->zFilename); */
   }
   sqliteFree(pPager);
   return SQLITE_OK;
@@ -575,13 +678,19 @@ int sqlitepager_ref(void *pData){
 ** the risk of having to do another fsync() later on.  Writing dirty
 ** free pages in this way was observed to make database operations go
 ** up to 10 times faster.
+**
+** If we are writing to temporary database, there is no need to preserve
+** the integrity of the journal file, so we can save time and skip the
+** fsync().
 */
 static int syncAllPages(Pager *pPager){
   PgHdr *pPg;
   int rc = SQLITE_OK;
   if( pPager->needSync ){
-    rc = sqliteOsSync(&pPager->jfd);
+    if( !pPager->tempFile ){
-    if( rc!=0 ) return rc;
+      rc = sqliteOsSync(&pPager->jfd);
+      if( rc!=0 ) return rc;
+    }
     pPager->needSync = 0;
   }
   for(pPg=pPager->pFirst; pPg; pPg=pPg->pNextFree){
@@ -774,6 +883,11 @@ int sqlitepager_get(Pager *pPager, Pgno pgno, void **ppPage){
     }else{
       pPg->inJournal = 0;
     }
+    if( pPager->aInCkpt && (int)pgno*SQLITE_PAGE_SIZE<=pPager->ckptSize ){
+      pPg->inCkpt = (pPager->aInCkpt[pgno/8] & (1<<(pgno&7)))!=0;
+    }else{
+      pPg->inCkpt = 0;
+    }
     pPg->dirty = 0;
     pPg->nRef = 1;
     REFINFO(pPg);
@@ -922,11 +1036,16 @@ int sqlitepager_write(void *pData){
   ** to the journal then we can return right away.
   */
   pPg->dirty = 1;
-  if( pPg->inJournal ){ return SQLITE_OK; }
+  if( pPg->inJournal && (pPg->inCkpt || pPager->ckptOpen==0) ){
+    return SQLITE_OK;
+  }
 
   /* If we get this far, it means that the page needs to be
-  ** written to the journal file. First check to see if the
+  ** written to the transaction journal or the ckeckpoint journal
-  ** journal exists and create it if it does not.
+  ** or both.
+  **
+  ** First check to see that the transaction journal exists and
+  ** create it if it does not.
   */
   assert( pPager->state!=SQLITE_UNLOCK );
   if( pPager->state==SQLITE_READLOCK ){
@@ -940,7 +1059,7 @@ int sqlitepager_write(void *pData){
       sqliteOsReadLock(&pPager->fd);
       return SQLITE_NOMEM;
     }
-    rc = sqliteOsOpenExclusive(pPager->zJournal, &pPager->jfd);
+    rc = sqliteOsOpenExclusive(pPager->zJournal, &pPager->jfd, 0);
     if( rc!=SQLITE_OK ){
       sqliteFree(pPager->aInJournal);
       pPager->aInJournal = 0;
@@ -965,10 +1084,11 @@ int sqlitepager_write(void *pData){
   assert( pPager->state==SQLITE_WRITELOCK );
   assert( pPager->journalOpen );
 
-  /* The journal now exists and we have a write lock on the
+  /* The transaction journal now exists and we have a write lock on the
-  ** main database file.  Write the current page to the journal.
+  ** main database file.  Write the current page to the transaction 
+  ** journal if it is not there already.
   */
-  if( (int)pPg->pgno <= pPager->origDbSize ){
+  if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
     rc = sqliteOsWrite(&pPager->jfd, &pPg->pgno, sizeof(Pgno));
     if( rc==SQLITE_OK ){
       rc = sqliteOsWrite(&pPager->jfd, pData, SQLITE_PAGE_SIZE);
@@ -981,11 +1101,35 @@ int sqlitepager_write(void *pData){
     assert( pPager->aInJournal!=0 );
     pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
     pPager->needSync = 1;
+    pPg->inJournal = 1;
+    if( pPager->ckptOpen ){
+      pPager->aInCkpt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+      pPg->inCkpt = 1;
+    }
   }
 
-  /* Mark the current page as being in the journal and return.
+  /* If the checkpoint journal is open and the page is not in it,
+  ** then write the current page to the checkpoint journal.
+  */
+  if( pPager->ckptOpen && !pPg->inCkpt 
+      && (int)pPg->pgno*SQLITE_PAGE_SIZE < pPager->ckptSize ){
+    assert( pPg->inJournal );
+    rc = sqliteOsWrite(&pPager->cpfd, &pPg->pgno, sizeof(Pgno));
+    if( rc==SQLITE_OK ){
+      rc = sqliteOsWrite(&pPager->cpfd, pData, SQLITE_PAGE_SIZE);
+    }
+    if( rc!=SQLITE_OK ){
+      sqlitepager_rollback(pPager);
+      pPager->errMask |= PAGER_ERR_FULL;
+      return rc;
+    }
+    assert( pPager->aInCkpt!=0 );
+    pPager->aInCkpt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+    pPg->inCkpt = 1;
+  }
+
+  /* Update the database size and return.
   */
-  pPg->inJournal = 1;
   if( pPager->dbSize<(int)pPg->pgno ){
     pPager->dbSize = pPg->pgno;
   }
@@ -1105,6 +1249,63 @@ int *sqlitepager_stats(Pager *pPager){
   return a;
 }
 
+/*
+** Set the checkpoint.
+**
+** This routine should be called with the transaction journal already
+** open.  A new checkpoint journal is created that can be used to rollback
+** changes of a single command within a larger transaction.
+*/
+int sqlitepager_ckpt_begin(Pager *pPager){
+  int rc;
+  char zTemp[SQLITE_TEMPNAME_SIZE];
+  assert( pPager->journalOpen );
+  assert( !pPager->ckptOpen );
+  pPager->aInCkpt = sqliteMalloc( pPager->dbSize/8 + 1 );
+  if( pPager->aInCkpt==0 ){
+    sqliteOsReadLock(&pPager->fd);
+    return SQLITE_NOMEM;
+  }
+  rc = sqliteOsFileSize(&pPager->jfd, &pPager->ckptJSize);
+  if( rc ) goto ckpt_begin_failed;
+  pPager->ckptSize = pPager->dbSize * SQLITE_PAGE_SIZE;
+  rc = sqlitepager_opentemp(zTemp, &pPager->cpfd);
+  if( rc ) goto ckpt_begin_failed;
+  pPager->ckptOpen = 1;
+  return SQLITE_OK;
+ 
+ckpt_begin_failed:
+  if( pPager->aInCkpt ){
+    sqliteFree(pPager->aInCkpt);
+    pPager->aInCkpt = 0;
+  }
+  return rc;
+}
+
+/*
+** Commit a checkpoint.
+*/
+int sqlitepager_ckpt_commit(Pager *pPager){
+  assert( pPager->ckptOpen );
+  sqliteOsClose(&pPager->cpfd);
+  sqliteFree(pPager->aInCkpt);
+  pPager->ckptOpen = 0;
+  return SQLITE_OK;
+}
+
+/*
+** Rollback a checkpoint.
+*/
+int sqlitepager_ckpt_rollback(Pager *pPager){
+  int rc;
+  assert( pPager->ckptOpen );
+  rc = pager_ckpt_playback(pPager);
+  sqliteOsClose(&pPager->cpfd);
+  sqliteFree(pPager->aInCkpt);
+  pPager->ckptOpen = 0;
+  return rc;
+}
+
 #if SQLITE_TEST
 /*
 ** Print a listing of all referenced pages and their ref count.

src/pager.h

@@ -13,7 +13,7 @@
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.13 2001/12/15 14:22:19 drh Exp $
+** @(#) $Id: pager.h,v 1.14 2002/02/02 15:01:16 drh Exp $
 */
 
 /*
@@ -62,6 +62,9 @@ int sqlitepager_pagecount(Pager*);
 int sqlitepager_commit(Pager*);
 int sqlitepager_rollback(Pager*);
 int sqlitepager_isreadonly(Pager*);
+int sqlitepager_ckpt_begin(Pager*);
+int sqlitepager_ckpt_commit(Pager*);
+int sqlitepager_ckpt_rollback(Pager*);
 int *sqlitepager_stats(Pager*);
 
 #ifdef SQLITE_TEST

src/parse.y

@@ -14,7 +14,7 @@
 ** the parser.  Lemon will also generate a header file containing
 ** numeric codes for all of the tokens.
 **
-** @(#) $Id: parse.y,v 1.45 2002/01/31 15:54:22 drh Exp $
+** @(#) $Id: parse.y,v 1.46 2002/02/02 15:01:16 drh Exp $
 */
 %token_prefix TK_
 %token_type {Token}
@@ -56,7 +56,11 @@ explain ::= EXPLAIN.    {pParse->explain = 1;}
 
 ///////////////////// Begin and end transactions. ////////////////////////////
 //
-cmd ::= BEGIN trans_opt onconf(R).  {sqliteBeginTransaction(pParse,R);}
+
+// For now, disable the ability to change the default conflict resolution
+// algorithm in a transaction.  We made add it back later.
+// cmd ::= BEGIN trans_opt onconf(R).  {sqliteBeginTransaction(pParse,R);}
+cmd ::= BEGIN trans_opt.        {sqliteBeginTransaction(pParse, OE_Default);}
 trans_opt ::= .
 trans_opt ::= TRANSACTION.
 trans_opt ::= TRANSACTION ids.
@@ -325,11 +329,15 @@ setlist(A) ::= ids(X) EQ expr(Y).   {A = sqliteExprListAppend(0,Y,&X);}
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= INSERT orconf(R) INTO ids(X) inscollist_opt(F) VALUES LP itemlist(Y) RP.
+cmd ::= insert_cmd(R) INTO ids(X) inscollist_opt(F) VALUES LP itemlist(Y) RP.
                {sqliteInsert(pParse, &X, Y, 0, F, R);}
-cmd ::= INSERT orconf(R) INTO ids(X) inscollist_opt(F) select(S).
+cmd ::= insert_cmd(R) INTO ids(X) inscollist_opt(F) select(S).
                {sqliteInsert(pParse, &X, 0, S, F, R);}
 
+%type insert_cmd {int}
+insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
+insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}
+
 
 %type itemlist {ExprList*}
 %destructor itemlist {sqliteExprListDelete($$);}

src/test2.c

@@ -13,7 +13,7 @@
 ** is not included in the SQLite library.  It is used for automated
 ** testing of the SQLite library.
 **
-** $Id: test2.c,v 1.6 2001/10/12 17:30:05 drh Exp $
+** $Id: test2.c,v 1.7 2002/02/02 15:01:16 drh Exp $
 */
 #include "sqliteInt.h"
 #include "pager.h"
@@ -159,6 +159,87 @@ static int pager_commit(
   return TCL_OK;
 }
 
+/*
+** Usage:   pager_ckpt_begin ID
+**
+** Start a new checkpoint.
+*/
+static int pager_ckpt_begin(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  char **argv            /* Text of each argument */
+){
+  Pager *pPager;
+  int rc;
+  if( argc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
+       " ID\"", 0);
+    return TCL_ERROR;
+  }
+  if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR;
+  rc = sqlitepager_ckpt_begin(pPager);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, errorName(rc), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
+/*
+** Usage:   pager_ckpt_rollback ID
+**
+** Rollback changes to a checkpoint
+*/
+static int pager_ckpt_rollback(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  char **argv            /* Text of each argument */
+){
+  Pager *pPager;
+  int rc;
+  if( argc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
+       " ID\"", 0);
+    return TCL_ERROR;
+  }
+  if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR;
+  rc = sqlitepager_ckpt_rollback(pPager);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, errorName(rc), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
+/*
+** Usage:   pager_ckpt_commit ID
+**
+** Commit changes to a checkpoint
+*/
+static int pager_ckpt_commit(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  char **argv            /* Text of each argument */
+){
+  Pager *pPager;
+  int rc;
+  if( argc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
+       " ID\"", 0);
+    return TCL_ERROR;
+  }
+  if( Tcl_GetInt(interp, argv[1], (int*)&pPager) ) return TCL_ERROR;
+  rc = sqlitepager_ckpt_commit(pPager);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, errorName(rc), 0);
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage:   pager_stats ID
 **
@@ -393,6 +474,9 @@ int Sqlitetest2_Init(Tcl_Interp *interp){
   Tcl_CreateCommand(interp, "pager_close", pager_close, 0, 0);
   Tcl_CreateCommand(interp, "pager_commit", pager_commit, 0, 0);
   Tcl_CreateCommand(interp, "pager_rollback", pager_rollback, 0, 0);
+  Tcl_CreateCommand(interp, "pager_ckpt_begin", pager_ckpt_begin, 0, 0);
+  Tcl_CreateCommand(interp, "pager_ckpt_commit", pager_ckpt_commit, 0, 0);
+  Tcl_CreateCommand(interp, "pager_ckpt_rollback", pager_ckpt_rollback, 0, 0);
   Tcl_CreateCommand(interp, "pager_stats", pager_stats, 0, 0);
   Tcl_CreateCommand(interp, "pager_pagecount", pager_pagecount, 0, 0);
   Tcl_CreateCommand(interp, "page_get", page_get, 0, 0);

www/changes.tcl

@@ -21,6 +21,10 @@ chng {2002 Jan 30 (2.3.0 beta)} {
 <li>Added the ability to resolve constraint conflicts is ways other than
     an abort and rollback.  See the documentation on the "ON CONFLICT"
     clause for details.</li>
+<li>Temporary files are now automatically deleted by the operating system
+    when closed.  There are no more dangling temporary files on a program
+    crash.  (If the OS crashes, fsck will delete the file after reboot 
+    under Unix.  I do not know what happens under Windows.)</li>
 <li>NOT NULL constraints are honored.</li>
 <li>The COPY command puts NULLs in columns whose data is '\N'.</li>
 <li>In the COPY command, backslash can now be used to escape a newline.</li>