Add the Bitvec object for tracking which pages have been journalled.

This reduces memory consumption and runs faster than the bitmap approach
it replaced. (CVS 4794)

FossilOrigin-Name: 7c57bdbcdb84d95419ec7029d2e13c593854a8d8

main.mk

@@ -48,7 +48,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src
 
 # Object files for the SQLite library.
 #
-LIBOBJ+= alter.o analyze.o attach.o auth.o btmutex.o btree.o build.o \
+LIBOBJ+= alter.o analyze.o attach.o auth.o bitvec.o btmutex.o btree.o build.o \
          callback.o complete.o date.o delete.o \
          expr.o fault.o func.o hash.o insert.o journal.o loadext.o \
          main.o malloc.o mem1.o mem2.o mem3.o mem4.o mem5.o \
@@ -85,6 +85,7 @@ SRC = \
   $(TOP)/src/analyze.c \
   $(TOP)/src/attach.c \
   $(TOP)/src/auth.c \
+  $(TOP)/src/bitvec.c \
   $(TOP)/src/btmutex.c \
   $(TOP)/src/btree.c \
   $(TOP)/src/btree.h \

manifest

@@ -1,5 +1,5 @@
-C The\spower-of-two\sfirst-fit\smemory\sallocator\sis\snow\sworking.\s(CVS\s4793)
+C Add\sthe\sBitvec\sobject\sfor\stracking\swhich\spages\shave\sbeen\sjournalled.\nThis\sreduces\smemory\sconsumption\sand\sruns\sfaster\sthan\sthe\sbitmap\sapproach\nit\sreplaced.\s(CVS\s4794)
-D 2008-02-16T16:21:46
+D 2008-02-18T14:47:34
 F Makefile.arm-wince-mingw32ce-gcc ac5f7b2cef0cd850d6f755ba6ee4ab961b1fadf7
 F Makefile.in bc2b5df3e3d0d4b801b824b7ef6dec43812b049b
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -64,7 +64,7 @@ F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387
-F main.mk e5649378177ca11d8a115a09e4284d14ffdc64d6
+F main.mk 648992e13f77f6039b2824bd97c8853beea20dbd
 F mkdll.sh 712e74f3efe08a6ba12b2945d018a29a89d7fe3b
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 1a866b53637dab137191341cc875575a5ca110fb
@@ -82,6 +82,7 @@ F src/alter.c b494a50f239a760565ce6220ee316e96956ec054
 F src/analyze.c a78ac494668581fe7f54ee63700815bb0ea34261
 F src/attach.c e13d62597e8725075b27186817f7e745122af24e
 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627
+F src/bitvec.c bc5b52a590dc38a48fdded1f098b84af673448c9
 F src/btmutex.c 483ced3c52205b04b97df69161fadbf87f4f1ea2
 F src/btree.c 29ea577155f39be65bdec1c7782301ff2ee9eb3f
 F src/btree.h 19dcf5ad23c17b98855da548e9a8e3eb4429d5eb
@@ -126,7 +127,7 @@ F src/os_unix.c e4daef7628f690fa2b188af3632fb18f96525946
 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
 F src/os_win.c c832d528ea774c7094d887749d71884984c9034c
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c 1960545a871f9b57a80e485e5969ee045b7a00d8
+F src/pager.c d9aeb0a131ca432f5cf06693a729d0ff818fc9c2
 F src/pager.h 8174615ffd14ccc2cad2b081b919a398fa95e3f9
 F src/parse.y 00f2698c8ae84f315be5e3f10b63c94f531fdd6d
 F src/pragma.c e3f39f8576234887ecd0c1de43dc51af5855930c
@@ -138,12 +139,12 @@ F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96
 F src/shell.c c1ef4eb7872afb7417e52d6ec3f2d15be90cba8a
 F src/sqlite.h.in 74e71510ce5967333a36329212eca0833f6300bd
 F src/sqlite3ext.h a93f59cdee3638dc0c9c086f80df743a4e68c3cb
-F src/sqliteInt.h c82511830758350ed4cedd0815add7cbb145e08d
+F src/sqliteInt.h 729101936ddcae387e39b2be0adcaa7ffed234d4
 F src/sqliteLimit.h ee4430f88f69bf63527967bb35ca52af7b0ccb1e
 F src/table.c 46ccf9b7892a86f57420ae7bac69ecd5e72d26b5
 F src/tclsqlite.c 0d4483e37c6a1e87f80565e50d977df6dd2bf732
 F src/test1.c 28b135491b436b1df6390a8b53834da2f94efca4
-F src/test2.c 77b34303883b9d722c65a6879bb0163a400e3789
+F src/test2.c 355d5693ca3ee705548fa7f795592a37b2372b70
 F src/test3.c 4557ee13c6e5921eb28979ff77cdbd913bfde6be
 F src/test4.c c2c0f5dc907f1346f5d4b65eb5799f11eb9e4071
 F src/test5.c 3a6a5717a149d7ca2e6d14f5be72cf7555d54dc4
@@ -210,6 +211,7 @@ F test/bigfile.test 9a6a8346e4042d9c781ed6cb6553ac871ae30618
 F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747
 F test/bind.test 261fd1603613e7f877a516d29f281c9d8c2ecf52
 F test/bindxfer.test b9a57f66dbd317feeefa28bd65b6576f1592ee98
+F test/bitvec.test 52a1caf5b4f037982f0e7720ffff6296f20940a6
 F test/blob.test f2dbdbf1159674283645c2636436839313ee7131
 F test/btree.test d22b1b2cc9becc36f6b1f2f91b9fca1e48060979
 F test/btree2.test 4b56a2a4a4f84d68c77aef271223a713bf5ebafc
@@ -549,7 +551,7 @@ F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8
 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf
 F tool/mkkeywordhash.c ef93810fc41fb3d3dbacf9a33a29be88ea99ffa9
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
-F tool/mksqlite3c.tcl c1876ef95be512ce466f09d7b2d1157f9766f2b1
+F tool/mksqlite3c.tcl a50378947f5bdad7b2d2d08960a0c7c52b7c47c1
 F tool/mksqlite3internalh.tcl 47737a925fb02fce43e2c0a14b3cc17574a4d44a
 F tool/omittest.tcl 7d1fdf469e2f4d175f70c36e469db64a1626fabb
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@@ -619,7 +621,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P dedf5f230bf34a207f2ee0a8349a2ea602a38aba
+P d134d29cea971eb01a0e0fd94341ab79e2d5b57a
-R cf36f2227f0aebc7182206911a7828ca
+R 7f46679eadc481e9280dcd7ae7ce6455
 U drh
-Z 7f0671ff9d8e2cc545c59a85da0cd625
+Z dc5ae8e1fa2e180c4555dfd11cef133e

manifest.uuid

@@ -1 +1 @@
-d134d29cea971eb01a0e0fd94341ab79e2d5b57a
+7c57bdbcdb84d95419ec7029d2e13c593854a8d8

src/bitvec.c

@@ -0,0 +1,208 @@
+/*
+** 2008 February 16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements an object that represents a fixed-length
+** bitmap.  Bits are numbered starting with 1.
+**
+** A bitmap is used to record what pages a database file have been
+** journalled during a transaction.  Usually only a few pages are
+** journalled.  So the bitmap is usually sparse and has low cardinality.
+** But sometimes (for example when during a DROP of a large table) most
+** or all of the pages get journalled.  In those cases, the bitmap becomes
+** dense.  The algorithm needs to handle both cases well.
+**
+** The size of the bitmap is fixed when the object is created.
+**
+** All bits are clear when the bitmap is created.  Individual bits
+** may be set or cleared one at a time.
+**
+** Test operations are about 100 times more common that set operations.
+** Clear operations are exceedingly rare.  There are usually between
+** 5 and 500 set operations per Bitvec object, though the number of sets can
+** sometimes grow into tens of thousands or larger.  The size of the
+** Bitvec object is the number of pages in the database file at the
+** start of a transaction, and is thus usually less than a few thousand,
+** but can be as large as 2 billion for a really big database.
+**
+** @(#) $Id: bitvec.c,v 1.1 2008/02/18 14:47:34 drh Exp $
+*/
+#include "sqliteInt.h"
+
+#define BITVEC_SZ        512
+#define BITVEC_NCHAR     (BITVEC_SZ-12)
+#define BITVEC_NBIT      (BITVEC_NCHAR*8)
+#define BITVEC_NINT      ((BITVEC_SZ-12)/4)
+#define BITVEC_MXHASH    (BITVEC_NINT/2)
+#define BITVEC_NPTR      ((BITVEC_SZ-12)/8)
+
+#define BITVEC_HASH(X)   (((X)*37)%BITVEC_NINT)
+
+/*
+** A bitmap is an instance of the following structure.
+**
+** This bitmap records the existance of zero or more bits
+** with values between 1 and iSize, inclusive.
+**
+** There are three possible representations of the bitmap.
+** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight
+** bitmap.  The least significant bit is bit 1.
+**
+** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is
+** a hash table that will hold up to BITVEC_MXHASH distinct values.
+**
+** Otherwise, the value i is redirected into one of BITVEC_NPTR
+** sub-bitmaps pointed to by Bitvec.u.apSub[].  Each subbitmap
+** handles up to iDivisor separate values of i.  apSub[0] holds
+** values between 1 and iDivisor.  apSub[1] holds values between
+** iDivisor+1 and 2*iDivisor.  apSub[N] holds values between
+** N*iDivisor+1 and (N+1)*iDivisor.  Each subbitmap is normalized
+** to hold deal with values between 1 and iDivisor.
+*/
+struct Bitvec {
+  u32 iSize;      /* Maximum bit index */
+  u32 nSet;       /* Number of bits that are set */
+  u32 iDivisor;   /* Number of bits handled by each apSub[] entry */
+  union {
+    u8 aBitmap[BITVEC_NCHAR];    /* Bitmap representation */
+    u32 aHash[BITVEC_NINT];      /* Hash table representation */
+    Bitvec *apSub[BITVEC_NPTR];  /* Recursive representation */
+  } u;
+};
+
+/*
+** Create a new bitmap object able to handle bits between 0 and iSize,
+** inclusive.  Return a pointer to the new object.  Return NULL if 
+** malloc fails.
+*/
+Bitvec *sqlite3BitvecCreate(u32 iSize){
+  Bitvec *p;
+  assert( sizeof(*p)==BITVEC_SZ );
+  p = sqlite3MallocZero( sizeof(*p) );
+  if( p ){
+    p->iSize = iSize;
+  }
+  return p;
+}
+
+/*
+** Check to see if the i-th bit is set.  Return true or false.
+** If p is NULL (if the bitmap has not been created) or if
+** i is out of range, then return false.
+*/
+int sqlite3BitvecTest(Bitvec *p, u32 i){
+  assert( i>0 );
+  if( p==0 ) return 0;
+  if( i>p->iSize ) return 0;
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    return (p->u.aBitmap[i/8] & (1<<(i&7)))!=0;
+  }
+  if( p->iDivisor>0 ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    return sqlite3BitvecTest(p->u.apSub[bin], i);
+  }else{
+    u32 h = BITVEC_HASH(i);
+    while( p->u.aHash[h] ){
+      if( p->u.aHash[h]==i ) return 1;
+      h++;
+      if( h>=BITVEC_NINT ) h = 0;
+    }
+    return 0;
+  }
+}
+
+/*
+** Set the i-th bit.  Return 0 on success and an error code if
+** anything goes wrong.
+*/
+int sqlite3BitvecSet(Bitvec *p, u32 i){
+  u32 h;
+  assert( p!=0 );
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    p->u.aBitmap[i/8] |= 1 << (i&7);
+    return SQLITE_OK;
+  }
+  if( p->iDivisor ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    if( p->u.apSub[bin]==0 ){
+      sqlite3FaultBenign(SQLITE_FAULTINJECTOR_MALLOC, 1);
+      p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor );
+      sqlite3FaultBenign(SQLITE_FAULTINJECTOR_MALLOC, 0);
+      if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM;
+    }
+    return sqlite3BitvecSet(p->u.apSub[bin], i);
+  }
+  h = BITVEC_HASH(i);
+  while( p->u.aHash[h] ){
+    if( p->u.aHash[h]==i ) return SQLITE_OK;
+    h++;
+    if( h==BITVEC_NINT ) h = 0;
+  }
+  p->nSet++;
+  if( p->nSet>=BITVEC_MXHASH ){
+    int j, rc;
+    u32 aiValues[BITVEC_NINT];
+    memcpy(aiValues, p->u.aHash, sizeof(aiValues));
+    memset(p->u.apSub, 0, sizeof(p->u.apSub[0])*BITVEC_NPTR);
+    p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR;
+    sqlite3BitvecSet(p, i);
+    for(rc=j=0; j<BITVEC_NINT; j++){
+      if( aiValues[j] ) rc |= sqlite3BitvecSet(p, aiValues[j]);
+    }
+    return rc;
+  }
+  p->u.aHash[h] = i;
+  return SQLITE_OK;
+}
+
+/*
+** Clear the i-th bit.  Return 0 on success and an error code if
+** anything goes wrong.
+*/
+void sqlite3BitvecClear(Bitvec *p, u32 i){
+  assert( p!=0 );
+  if( p->iSize<=BITVEC_NBIT ){
+    i--;
+    p->u.aBitmap[i/8] &= ~(1 << (i&7));
+  }else if( p->iDivisor ){
+    u32 bin = (i-1)/p->iDivisor;
+    i = (i-1)%p->iDivisor + 1;
+    if( p->u.apSub[bin] ){
+      sqlite3BitvecClear(p->u.apSub[bin], i);
+    }
+  }else{
+    int j;
+    u32 aiValues[BITVEC_NINT];
+    memcpy(aiValues, p->u.aHash, sizeof(aiValues));
+    memset(p->u.aHash, 0, sizeof(p->u.aHash[0])*BITVEC_NINT);
+    p->nSet = 0;
+    for(j=0; j<BITVEC_NINT; j++){
+      if( aiValues[j] && aiValues[j]!=i ) sqlite3BitvecSet(p, aiValues[j]);
+    }
+  }
+}
+
+/*
+** Destroy a bitmap object.  Reclaim all memory used.
+*/
+void sqlite3BitvecDestroy(Bitvec *p){
+  if( p==0 ) return;
+  if( p->iDivisor ){
+    int i;
+    for(i=0; i<BITVEC_NPTR; i++){
+      sqlite3BitvecDestroy(p->u.apSub[i]);
+    }
+  }
+  sqlite3_free(p);
+}

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.406 2008/02/14 23:26:56 drh Exp $
+** @(#) $Id: pager.c,v 1.407 2008/02/18 14:47:34 drh Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -213,9 +213,9 @@ struct PagerLruLink {
 **     has been synced to disk.  For pages that are in the original
 **     database file, the following expression should always be true:
 **
-**       inJournal = (pPager->aInJournal[(pgno-1)/8] & (1<<((pgno-1)%8))!=0
+**       inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno)
 **
-**     The pPager->aInJournal[] array is only valid for the original
+**     The pPager->pInJournal object is only valid for the original
 **     pages of the database, not new pages that are added to the end
 **     of the database, so obviously the above expression cannot be
 **     valid for new pages.  For new pages inJournal is always 0.
@@ -365,8 +365,8 @@ struct Pager {
   int nRef;                   /* Number of in-memory pages with PgHdr.nRef>0 */
   int mxPage;                 /* Maximum number of pages to hold in cache */
   Pgno mxPgno;                /* Maximum allowed size of the database */
-  u8 *aInJournal;             /* One bit for each page in the database file */
+  Bitvec *pInJournal;         /* One bit for each page in the database file */
-  u8 *aInStmt;                /* One bit for each page in the database */
+  Bitvec *pInStmt;            /* One bit for each page in the database */
   char *zFilename;            /* Name of the database file */
   char *zJournal;             /* Name of the journal file */
   char *zDirectory;           /* Directory hold database and journal files */
@@ -666,9 +666,7 @@ static int pageInStatement(PgHdr *pPg){
   if( MEMDB ){
     return PGHDR_TO_HIST(pPg, pPager)->inStmt;
   }else{
-    Pgno pgno = pPg->pgno;
+    return sqlite3BitvecTest(pPager->pInStmt, pPg->pgno);
-    u8 *a = pPager->aInStmt;
-    return (a && (int)pgno<=pPager->stmtSize && (a[pgno/8] & (1<<(pgno&7))));
   }
 }
 
@@ -1252,14 +1250,14 @@ static void pager_unlock(Pager *pPager){
         pager_reset(pPager);
         if( pPager->stmtOpen ){
           sqlite3OsClose(pPager->stfd);
-          sqlite3_free(pPager->aInStmt);
+          sqlite3BitvecDestroy(pPager->pInStmt);
-          pPager->aInStmt = 0;
+          pPager->pInStmt = 0;
         }
         if( pPager->journalOpen ){
           sqlite3OsClose(pPager->jfd);
           pPager->journalOpen = 0;
-          sqlite3_free(pPager->aInJournal);
+          sqlite3BitvecDestroy(pPager->pInJournal);
-          pPager->aInJournal = 0;
+          pPager->pInJournal = 0;
         }
         pPager->stmtOpen = 0;
         pPager->stmtInUse = 0;
@@ -1334,8 +1332,8 @@ static int pager_end_transaction(Pager *pPager){
         rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0);
       }
     }
-    sqlite3_free( pPager->aInJournal );
+    sqlite3BitvecDestroy(pPager->pInJournal);
-    pPager->aInJournal = 0;
+    pPager->pInJournal = 0;
     for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
       pPg->inJournal = 0;
       pPg->dirty = 0;
@@ -1349,7 +1347,7 @@ static int pager_end_transaction(Pager *pPager){
     pPager->dirtyCache = 0;
     pPager->nRec = 0;
   }else{
-    assert( pPager->aInJournal==0 );
+    assert( pPager->pInJournal==0 );
     assert( pPager->dirtyCache==0 || pPager->useJournal==0 );
   }
 
@@ -2661,7 +2659,7 @@ int sqlite3PagerClose(Pager *pPager){
   if( pPager->journalOpen ){
     sqlite3OsClose(pPager->jfd);
   }
-  sqlite3_free(pPager->aInJournal);
+  sqlite3BitvecDestroy(pPager->pInJournal);
   if( pPager->stmtOpen ){
     sqlite3OsClose(pPager->stfd);
   }
@@ -3625,17 +3623,8 @@ static int pagerAcquire(
 
     pPg->pgno = pgno;
     assert( !MEMDB || pgno>pPager->stmtSize );
-    if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
+    pPg->inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno);
-#if 0
-      sqlite3CheckMemory(pPager->aInJournal, pgno/8);
-#endif
-      assert( pPager->journalOpen );
-      pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0;
     pPg->needSync = 0;
-    }else{
-      pPg->inJournal = 0;
-      pPg->needSync = 0;
-    }
 
     makeClean(pPg);
     pPg->nRef = 1;
@@ -3801,12 +3790,12 @@ static int pager_open_journal(Pager *pPager){
   assert( pPager->state>=PAGER_RESERVED );
   assert( pPager->journalOpen==0 );
   assert( pPager->useJournal );
-  assert( pPager->aInJournal==0 );
+  assert( pPager->pInJournal==0 );
   sqlite3PagerPagecount(pPager);
   pagerLeave(pPager);
-  pPager->aInJournal = sqlite3MallocZero( pPager->dbSize/8 + 1 );
+  pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize);
   pagerEnter(pPager);
-  if( pPager->aInJournal==0 ){
+  if( pPager->pInJournal==0 ){
     rc = SQLITE_NOMEM;
     goto failed_to_open_journal;
   }
@@ -3858,8 +3847,8 @@ static int pager_open_journal(Pager *pPager){
   return rc;
 
 failed_to_open_journal:
-  sqlite3_free(pPager->aInJournal);
+  sqlite3BitvecDestroy(pPager->pInJournal);
-  pPager->aInJournal = 0;
+  pPager->pInJournal = 0;
   return rc;
 }
 
@@ -3897,7 +3886,7 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){
   assert( pPg->nRef>0 );
   assert( pPager->state!=PAGER_UNLOCK );
   if( pPager->state==PAGER_SHARED ){
-    assert( pPager->aInJournal==0 );
+    assert( pPager->pInJournal==0 );
     if( MEMDB ){
       pPager->state = PAGER_EXCLUSIVE;
       pPager->origDbSize = pPager->dbSize;
@@ -3927,12 +3916,12 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){
     */
     assert( pPager->nRec==0 );
     assert( pPager->origDbSize==0 );
-    assert( pPager->aInJournal==0 );
+    assert( pPager->pInJournal==0 );
     sqlite3PagerPagecount(pPager);
     pagerLeave(pPager);
-    pPager->aInJournal = sqlite3MallocZero( pPager->dbSize/8 + 1 );
+    pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize );
     pagerEnter(pPager);
-    if( !pPager->aInJournal ){
+    if( !pPager->pInJournal ){
       rc = SQLITE_NOMEM;
     }else{
       pPager->origDbSize = pPager->dbSize;
@@ -4108,11 +4097,11 @@ static int pager_write(PgHdr *pPg){
           }
 
           pPager->nRec++;
-          assert( pPager->aInJournal!=0 );
+          assert( pPager->pInJournal!=0 );
-          pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+          sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
           pPg->needSync = !pPager->noSync;
           if( pPager->stmtInUse ){
-            pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+            sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
           }
         }
       }else{
@@ -4157,8 +4146,8 @@ static int pager_write(PgHdr *pPg){
           return rc;
         }
         pPager->stmtNRec++;
-        assert( pPager->aInStmt!=0 );
+        assert( pPager->pInStmt!=0 );
-        pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+        sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
       }
     }
   }
@@ -4227,9 +4216,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){
     for(ii=0; ii<nPage && rc==SQLITE_OK; ii++){
       Pgno pg = pg1+ii;
       PgHdr *pPage;
-      if( !pPager->aInJournal || pg==pPg->pgno || 
+      if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){
-          pg>pPager->origDbSize || !(pPager->aInJournal[pg/8]&(1<<(pg&7)))
-      ) {
         if( pg!=PAGER_MJ_PGNO(pPager) ){
           rc = sqlite3PagerGet(pPager, pg, &pPage);
           if( rc==SQLITE_OK ){
@@ -4403,13 +4390,13 @@ void sqlite3PagerDontRollback(DbPage *pPg){
   */
   assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize );
 
-  assert( pPager->aInJournal!=0 );
+  assert( pPager->pInJournal!=0 );
-  pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+  sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
   pPg->inJournal = 1;
   pPg->needRead = 0;
   if( pPager->stmtInUse ){
     assert( pPager->stmtSize <= pPager->origDbSize );
-    pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
+    sqlite3BitvecSet(pPager->pInStmt, pPg->pgno);
   }
   PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
   IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
@@ -4545,7 +4532,7 @@ int sqlite3PagerCommitPhaseOne(Pager *pPager, const char *zMaster, Pgno nTrunc){
         Pgno i;
         int iSkip = PAGER_MJ_PGNO(pPager);
         for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){
-          if( !(pPager->aInJournal[i/8] & (1<<(i&7))) && i!=iSkip ){
+          if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){
             rc = sqlite3PagerGet(pPager, i, &pPg);
             if( rc!=SQLITE_OK ) goto sync_exit;
             rc = sqlite3PagerWrite(pPg);
@@ -4803,10 +4790,10 @@ static int pagerStmtBegin(Pager *pPager){
   }
   assert( pPager->journalOpen );
   pagerLeave(pPager);
-  assert( pPager->aInStmt==0 );
+  assert( pPager->pInStmt==0 );
-  pPager->aInStmt = sqlite3MallocZero( pPager->dbSize/8 + 1 );
+  pPager->pInStmt = sqlite3BitvecCreate(pPager->dbSize);
   pagerEnter(pPager);
-  if( pPager->aInStmt==0 ){
+  if( pPager->pInStmt==0 ){
     /* sqlite3OsLock(pPager->fd, SHARED_LOCK); */
     return SQLITE_NOMEM;
   }
@@ -4832,9 +4819,9 @@ static int pagerStmtBegin(Pager *pPager){
   return SQLITE_OK;
  
 stmt_begin_failed:
-  if( pPager->aInStmt ){
+  if( pPager->pInStmt ){
-    sqlite3_free(pPager->aInStmt);
+    sqlite3BitvecDestroy(pPager->pInStmt);
-    pPager->aInStmt = 0;
+    pPager->pInStmt = 0;
   }
   return rc;
 }
@@ -4856,8 +4843,8 @@ int sqlite3PagerStmtCommit(Pager *pPager){
     PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
     if( !MEMDB ){
       /* sqlite3OsTruncate(pPager->stfd, 0); */
-      sqlite3_free( pPager->aInStmt );
+      sqlite3BitvecDestroy(pPager->pInStmt);
-      pPager->aInStmt = 0;
+      pPager->pInStmt = 0;
     }else{
       for(pPg=pPager->pStmt; pPg; pPg=pNext){
         PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
@@ -5027,12 +5014,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno){
   }else{
     pPg->needSync = 0;
   }
-  if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
+  pPg->inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno);
-    pPg->inJournal =  (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0;
-  }else{
-    pPg->inJournal = 0;
-    assert( pPg->needSync==0 || (int)pgno>pPager->origDbSize );
-  }
 
   /* Change the page number for pPg and insert it into the new hash-chain. */
   assert( pgno!=0 );
@@ -5053,11 +5035,11 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno){
     /* If needSyncPgno is non-zero, then the journal file needs to be 
     ** sync()ed before any data is written to database file page needSyncPgno.
     ** Currently, no such page exists in the page-cache and the 
-    ** Pager.aInJournal bit has been set. This needs to be remedied by loading
+    ** Pager.pInJournal bit has been set. This needs to be remedied by loading
     ** the page into the pager-cache and setting the PgHdr.needSync flag.
     **
     ** If the attempt to load the page into the page-cache fails, (due
-    ** to a malloc() or IO failure), clear the bit in the aInJournal[]
+    ** to a malloc() or IO failure), clear the bit in the pInJournal[]
     ** array. Otherwise, if the page is loaded and written again in
     ** this transaction, it may be written to the database file before
     ** it is synced into the journal file. This way, it may end up in
@@ -5071,8 +5053,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno){
     assert( pPager->needSync );
     rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
     if( rc!=SQLITE_OK ){
-      if( pPager->aInJournal && (int)needSyncPgno<=pPager->origDbSize ){
+      if( pPager->pInJournal && (int)needSyncPgno<=pPager->origDbSize ){
-        pPager->aInJournal[needSyncPgno/8] &= ~(1<<(needSyncPgno&7));
+        sqlite3BitvecClear(pPager->pInJournal, needSyncPgno);
       }
       pagerLeave(pPager);
       return rc;

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.662 2008/02/14 23:26:56 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.663 2008/02/18 14:47:34 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -363,6 +363,7 @@ struct BusyHandler {
 */
 typedef struct AggInfo AggInfo;
 typedef struct AuthContext AuthContext;
+typedef struct Bitvec Bitvec;
 typedef struct CollSeq CollSeq;
 typedef struct Column Column;
 typedef struct Db Db;
@@ -1759,6 +1760,12 @@ void sqlite3AddDefaultValue(Parse*,Expr*);
 void sqlite3AddCollateType(Parse*, Token*);
 void sqlite3EndTable(Parse*,Token*,Token*,Select*);
 
+Bitvec *sqlite3BitvecCreate(u32);
+int sqlite3BitvecTest(Bitvec*, u32);
+int sqlite3BitvecSet(Bitvec*, u32);
+void sqlite3BitvecClear(Bitvec*, u32);
+void sqlite3BitvecDestroy(Bitvec*);
+
 void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
 
 #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)

src/test2.c

@@ -13,7 +13,7 @@
 ** is not included in the SQLite library.  It is used for automated
 ** testing of the SQLite library.
 **
-** $Id: test2.c,v 1.52 2007/09/03 15:19:35 drh Exp $
+** $Id: test2.c,v 1.53 2008/02/18 14:47:34 drh Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -561,6 +561,104 @@ static int fake_big_file(
 }
 #endif
 
+/*
+**   sqlite3BitvecCreate SIZE
+**   sqlite3BitvecTest POINTER N
+**   sqlite3BitvecSet POINTER N
+**   sqlite3BitvecClear POINTER N
+**   sqlite3BitvecDestroy POINTER
+*/
+static int testBitvecCreate(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  const char **argv      /* Text of each argument */
+){
+  int size;
+  Bitvec *p;
+  char zBuf[100];
+  if( argc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " N\"", 
+           (void*)0);
+  }
+  if( Tcl_GetInt(interp, argv[1], &size) ) return TCL_ERROR;
+  p = sqlite3BitvecCreate(size);
+  sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",p);
+  Tcl_AppendResult(interp, zBuf, 0);
+  return TCL_OK;
+}  
+static int testBitvecTest(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  const char **argv      /* Text of each argument */
+){
+  int N;
+  Bitvec *p;
+  char zBuf[100];
+  if( argc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " PTR N\"", 
+           (void*)0);
+  }
+  p = (Bitvec*)sqlite3TextToPtr(argv[1]);
+  if( Tcl_GetInt(interp, argv[2], &N) ) return TCL_ERROR;
+  sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",sqlite3BitvecTest(p,N));
+  Tcl_AppendResult(interp, zBuf, 0);
+  return TCL_OK;
+}  
+static int testBitvecSet(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  const char **argv      /* Text of each argument */
+){
+  int N;
+  Bitvec *p;
+  char zBuf[100];
+  if( argc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " PTR N\"", 
+           (void*)0);
+  }
+  p = (Bitvec*)sqlite3TextToPtr(argv[1]);
+  if( Tcl_GetInt(interp, argv[2], &N) ) return TCL_ERROR;
+  sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",sqlite3BitvecSet(p,N));
+  Tcl_AppendResult(interp, zBuf, 0);
+  return TCL_OK;
+}  
+static int testBitvecClear(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  const char **argv      /* Text of each argument */
+){
+  int N;
+  Bitvec *p;
+  if( argc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " PTR N\"", 
+           (void*)0);
+  }
+  p = (Bitvec*)sqlite3TextToPtr(argv[1]);
+  if( Tcl_GetInt(interp, argv[2], &N) ) return TCL_ERROR;
+  sqlite3BitvecClear(p,N);
+  return TCL_OK;
+}  
+static int testBitvecDestroy(
+  void *NotUsed,
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int argc,              /* Number of arguments */
+  const char **argv      /* Text of each argument */
+){
+  Bitvec *p;
+  if( argc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " PTR\"", 
+           (void*)0);
+  }
+  p = (Bitvec*)sqlite3TextToPtr(argv[1]);
+  sqlite3BitvecDestroy(p);
+  return TCL_OK;
+}  
+       
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -594,6 +692,11 @@ int Sqlitetest2_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_DISKIO
     { "fake_big_file",           (Tcl_CmdProc*)fake_big_file       },
 #endif
+    { "sqlite3BitvecCreate",     (Tcl_CmdProc*)testBitvecCreate    },
+    { "sqlite3BitvecTest",       (Tcl_CmdProc*)testBitvecTest      },
+    { "sqlite3BitvecSet",        (Tcl_CmdProc*)testBitvecSet       },
+    { "sqlite3BitvecClear",      (Tcl_CmdProc*)testBitvecClear     },
+    { "sqlite3BitvecDestroy",    (Tcl_CmdProc*)testBitvecDestroy   },
   };
   int i;
   for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){

test/bitvec.test

@@ -0,0 +1,207 @@
+# 2008 February 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Unit testing of the Bitvec object.
+#
+# $Id: bitvec.test,v 1.1 2008/02/18 14:47:34 drh Exp $
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+#ifcapable !subquery {
+#  finish_test
+#  return
+#}
+
+do_test bitvec-1.1 {
+  set PTR [sqlite3BitvecCreate 4000]
+  for {set i 1} {$i<=4000} {incr i} {
+    if {$i%1000==999} continue
+    sqlite3BitvecSet $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=4000} {incr i} {
+    if {![sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {999 1999 2999 3999}
+do_test bitvec-1.2 {
+  set PTR [sqlite3BitvecCreate 4001]
+  for {set i 1} {$i<=4001} {incr i} {
+    if {$i%1000==999} continue
+    sqlite3BitvecSet $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=4001} {incr i} {
+    if {![sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {999 1999 2999 3999}
+do_test bitvec-1.3 {
+  set PTR [sqlite3BitvecCreate 40000]
+  for {set i 1} {$i<=40000} {incr i} {
+    if {$i%10000==9999} continue
+    sqlite3BitvecSet $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=40000} {incr i} {
+    if {![sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {9999 19999 29999 39999}
+do_test bitvec-1.4 {
+  set PTR [sqlite3BitvecCreate 2000000000]
+  for {set i 1000000} {$i<=1001000} {incr i} {
+    if {$i%1000==789} continue
+    sqlite3BitvecSet $PTR $i
+  }
+  set r {}
+  for {set i 1000000} {$i<=1001000} {incr i} {
+    if {![sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {1000789}
+
+do_test bitvec-2.1 {
+  set PTR [sqlite3BitvecCreate 4000]
+  for {set i 1} {$i<=4000} {incr i 2} {
+    sqlite3BitvecSet $PTR $i
+  }
+  for {set i 1} {$i<=4000} {incr i} {
+    sqlite3BitvecClear $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=4000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {}
+do_test bitvec-2.2 {
+  set PTR [sqlite3BitvecCreate 4001]
+  for {set i 1} {$i<=101} {incr i 2} {
+    sqlite3BitvecSet $PTR $i
+  }
+  for {set i 1} {$i<=99} {incr i} {
+    sqlite3BitvecClear $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=4000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {101}
+do_test bitvec-2.3 {
+  set PTR [sqlite3BitvecCreate 4001]
+  for {set i 1} {$i<=101} {incr i} {
+    sqlite3BitvecSet $PTR $i
+  }
+  for {set i 1} {$i<=99} {incr i} {
+    sqlite3BitvecClear $PTR $i
+  }
+  set r {}
+  for {set i 1} {$i<=4000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {100 101}
+do_test bitvec-2.4 {
+  set PTR [sqlite3BitvecCreate 5000]
+  for {set i 1} {$i<=5000} {incr i} {
+    sqlite3BitvecSet $PTR $i
+    if {$i%1000!=456} {sqlite3BitvecClear $PTR $i}
+  }
+  set r {}
+  for {set i 1} {$i<=5000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {456 1456 2456 3456 4456}
+
+do_test bitvec-3.1 {
+  set PTR [sqlite3BitvecCreate 2000000000]
+  for {set i 2000000} {$i<=3000000} {incr i 100000} {
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecSet $PTR $i
+    }
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecClear $PTR $i
+    }
+  }
+  set r {}
+  for {set i 2000000} {$i<=3000000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {}
+do_test bitvec-3.2 {
+  set PTR [sqlite3BitvecCreate 200000]
+  for {set i 1000} {$i<=190000} {incr i 10000} {
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecSet $PTR $i
+    }
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecClear $PTR $i
+    }
+  }
+  set r {}
+  for {set i 1} {$i<=200000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {}
+do_test bitvec-3.3 {
+  set PTR [sqlite3BitvecCreate 200000]
+  for {set i 1000} {$i<=190000} {incr i 10000} {
+    for {set j $i} {$j<=$i+500} {incr j} {
+      sqlite3BitvecSet $PTR $i
+    }
+    for {set j $i} {$j<=$i+500} {incr j} {
+      sqlite3BitvecClear $PTR $i
+    }
+  }
+  set r {}
+  for {set i 1} {$i<=200000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {}
+do_test bitvec-3.4 {
+  set PTR [sqlite3BitvecCreate 2000]
+  for {set i 10} {$i<=1900} {incr i 100} {
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecSet $PTR $i
+    }
+    for {set j $i} {$j<=$i+50} {incr j} {
+      sqlite3BitvecClear $PTR $i
+    }
+  }
+  set r {}
+  for {set i 1} {$i<=2000} {incr i} {
+    if {[sqlite3BitvecTest $PTR $i]} {lappend r $i}
+  }
+  sqlite3BitvecDestroy $PTR
+  set r
+} {}
+
+
+finish_test

tool/mksqlite3c.tcl

@@ -225,6 +225,7 @@ foreach file {
    os_unix.c
    os_win.c
 
+   bitvec.c
    pager.c
 
    btmutex.c