Separate the concepts of underlying storage size and mapped size in the

VFS shared-memory implementation.

FossilOrigin-Name: 4cbe49f13fed288f94ff305bcfd99df907bf7baf

manifest

@@ -1,5 +1,8 @@
-C Add\sa\smissing\swalIndexUnmap()\scall\sto\sthe\scheckpoint\scode.\sChange\sa\scouple\sof\sSQLITE_CANTOPEN\sconstants\sto\sSQLITE_CANTOPEN_BKPT.
+-----BEGIN PGP SIGNED MESSAGE-----
-D 2010-04-30T11:43:29
+Hash: SHA1
+
+C Separate\sthe\sconcepts\sof\sunderlying\sstorage\ssize\sand\smapped\ssize\sin\sthe\nVFS\sshared-memory\simplementation.
+D 2010-04-30T14:39:51
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306
 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c
 F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d
 F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0
-F src/os_unix.c 7451a717574c9b9809ad7c429345482c29239230
+F src/os_unix.c b2322d8fb389b4bc7834b1596bd8afac776a2f20
 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1
 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2
 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c
@@ -166,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9
 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b
 F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4
-F src/sqlite.h.in a710846e91159742ee3ddcbabb56eb7c7478e3a5
+F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f
 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89
 F src/sqliteInt.h 32dfe6294b46914fb567db493994bf902fb0f8c9
 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6
@@ -221,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
 F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
-F src/wal.c 9327e0d4e2b19dcf5614d3bb8cdff25ebbe0e290
+F src/wal.c a4c643582fd9956b46dca187161d466f584b47af
 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885
@@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 2e7a0050e192dd197d4db92393ab120897c14836
+P 1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161
-R c2c9e5e0c48b0291d2627d6aeb69855a
+R 2fbe76e11b4a2b6e620f52387007559a
-U dan
+U drh
-Z e7fa4c17c25ce2a31997877294b65d50
+Z 868ea682fbe4fab342b578dc69caee7c
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQFL2uu6oxKgR168RlERAqI3AJ4jq6FSOAaisShm+qyJakBJL+pVmwCgg1Vd
+tQBOoEYMwVK9lQZSibwuRzM=
+=5suj
+-----END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161
+4cbe49f13fed288f94ff305bcfd99df907bf7baf

src/os_unix.c

@@ -4594,11 +4594,9 @@ struct unixShmFile {
   sqlite3_mutex *mutexBuf;   /* Mutex to access zBuf[] */
   sqlite3_mutex *mutexRecov; /* The RECOVER mutex */
   char *zFilename;           /* Name of the file */
-  int size;                  /* Size of the file */
   int h;                     /* Open file descriptor */
-  char *pMMapBuf;            /* Where currently mmapped() */
+  int szMap;                 /* Size of the mapping of file into memory */
-  int nReadPrefix;           /* Number of SQLITE_SHM_READ_PREFIX locks */
+  char *pMMapBuf;            /* Where currently mmapped().  NULL if unmapped */
-  int nReadFull;             /* Number of SQLITE_SHM_READ_FULL locks */
   int nRef;                  /* Number of unixShm objects pointing to this */
   unixShm *pFirst;           /* All unixShm objects pointing to this */
   unixShmFile *pNext;        /* Next in list of all unixShmFile objects */
@@ -4995,8 +4993,6 @@ static int unixShmOpen(
     }
     pFile->fid.dev = sStat.st_dev;
     pFile->fid.ino = sStat.st_ino;
-    pFile->size = (int)sStat.st_size;
-    pFile->size = (pFile->size/SQLITE_UNIX_SHM_INCR)*SQLITE_UNIX_SHM_INCR;
 
     /* Check to see if another process is holding the dead-man switch.
     ** If not, truncate the file to zero length. 
@@ -5010,7 +5006,6 @@ static int unixShmOpen(
         rc = SQLITE_IOERR;
         goto shm_open_err;
       }
-      pFile->size = 0;
     }
     rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS);
     if( rc ) goto shm_open_err;
@@ -5078,22 +5073,64 @@ static int unixShmClose(sqlite3_shm *pSharedMem){
 }
 
 /*
-** Query and/or changes the size of a shared-memory segment.
+** Query and/or changes the size of the underlying storage for
-** The reqSize parameter is the new size of the segment, or -1 to
+** a shared-memory segment.  The reqSize parameter is the new size
-** do just a query.  The size of the segment after resizing is
+** of the underlying storage, or -1 to do just a query.  The size
-** written into pNewSize.  A writer lock is held on the shared memory
+** of the underlying storage (after resizing if resizing occurs) is
-** segment while resizing it.
+** written into pNewSize.
 **
-** If ppBuffer is not NULL, the a reader lock is acquired on the shared
+** This routine does not (necessarily) change the size of the mapping 
-** memory segment and *ppBuffer is made to point to the start of the 
+** of the underlying storage into memory.  Use xShmGet() to change
-** shared memory segment.  xShmRelease() must be called to release the
+** the mapping size.
-** lock.
+**
+** The reqSize parameter is the minimum size requested.  The implementation
+** is free to expand the storage to some larger amount if it chooses.
 */
 static int unixShmSize(
   sqlite3_shm *pSharedMem,  /* Pointer returned by unixShmOpen() */
   int reqSize,              /* Requested size.  -1 for query only */
-  int *pNewSize,            /* Write new size here */
+  int *pNewSize             /* Write new size here */
-  void **ppBuf              /* Write new buffer origin here */
+){
+  unixShm *p = (unixShm*)pSharedMem;
+  unixShmFile *pFile = p->pFile;
+  int rc = SQLITE_OK;
+  struct stat sStat;
+
+  if( reqSize>=0 ){
+    reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR;
+    reqSize *= SQLITE_UNIX_SHM_INCR;
+    rc = ftruncate(pFile->h, reqSize);
+  }
+  if( fstat(pFile->h, &sStat)==0 ){
+    *pNewSize = (int)sStat.st_size;
+  }else{
+    *pNewSize = 0;
+    rc = SQLITE_IOERR;
+  }
+  return rc;
+}
+
+
+/*
+** Map the shared storage into memory.  The minimum size of the
+** mapping should be reqMapSize if reqMapSize is positive.  If
+** reqMapSize is zero or negative, the implementation can choose
+** whatever mapping size is convenient.
+**
+** *ppBuf is made to point to the memory which is a mapping of the
+** underlying storage.  This segment is locked.  unixShmRelease()
+** must be called to release the lock.
+**
+** *pNewMapSize is set to the size of the mapping.
+**
+** *ppBuf and *pNewMapSize might be NULL and zero if no space has
+** yet been allocated to the underlying storage.
+*/
+static int unixShmGet(
+  sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */
+  int reqMapSize,          /* Requested size of mapping. -1 means don't care */
+  int *pNewMapSize,        /* Write new size of mapping here */
+  void **ppBuf             /* Write mapping buffer origin here */
 ){
   unixShm *p = (unixShm*)pSharedMem;
   unixShmFile *pFile = p->pFile;
@@ -5101,23 +5138,21 @@ static int unixShmSize(
 
   sqlite3_mutex_enter(pFile->mutexBuf);
   sqlite3_mutex_enter(pFile->mutex);
-  if( reqSize>=0 ){
+  if( pFile->szMap==0 || reqMapSize>pFile->szMap ){
-    reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR;
+    int actualSize;
-    reqSize *= SQLITE_UNIX_SHM_INCR;
+    if( unixShmSize(pSharedMem, -1, &actualSize)==SQLITE_OK
-    if( reqSize!=pFile->size ){
+     && reqMapSize<actualSize
-      if( pFile->pMMapBuf ) munmap(pFile->pMMapBuf, pFile->size);
+    ){
-      rc = ftruncate(pFile->h, reqSize);
+      reqMapSize = actualSize;
-      if( rc ){
+    }
-        pFile->pMMapBuf = 0;
+    if( pFile->pMMapBuf ){
-        pFile->size = 0;
+      munmap(pFile->pMMapBuf, pFile->szMap);
-      }else{
+    }
-        pFile->pMMapBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED,
+    pFile->pMMapBuf = mmap(0, reqMapSize, PROT_READ|PROT_WRITE, MAP_SHARED,
                            pFile->h, 0);
-        pFile->size = pFile->pMMapBuf ? reqSize : 0;
+    pFile->szMap = pFile->pMMapBuf ? reqMapSize : 0;
   }
-    }
+  *pNewMapSize = pFile->szMap;
-  }
-  *pNewSize = pFile->size;
   *ppBuf = pFile->pMMapBuf;
   sqlite3_mutex_leave(pFile->mutex);
   return rc;
@@ -6490,6 +6525,7 @@ int sqlite3_os_init(void){
     unixGetLastError,     /* xGetLastError */               \
     unixShmOpen,          /* xShmOpen */                    \
     unixShmSize,          /* xShmSize */                    \
+    unixShmGet,           /* xShmGet */                     \
     unixShmRelease,       /* xShmRelease */                 \
     0,                    /* xShmPush */                    \
     0,                    /* xShmPull */                    \

src/sqlite.h.in

@@ -844,7 +844,8 @@ struct sqlite3_vfs {
   ** definition.  Those that follow are added in version 2 or later
   */
   int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**);
-  int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, void**);
+  int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize);
+  int (*xShmGet)(sqlite3_shm*, int reqMapSize, int *pMapSize, void**);
   int (*xShmRelease)(sqlite3_shm*);
   int (*xShmPush)(sqlite3_shm*);
   int (*xShmPull)(sqlite3_shm*);

src/wal.c

@@ -126,7 +126,7 @@ struct Wal {
   sqlite3_file *pFd;         /* File handle for WAL file */
   u32 iCallback;             /* Value to pass to log callback (or 0) */
   sqlite3_shm *pWIndex;      /* The open wal-index file */
-  int szWIndex;              /* Size of the wal-index */
+  int szWIndex;              /* Size of the wal-index that is mapped in mem */
   u32 *pWiData;              /* Pointer to wal-index content in memory */
   u8 lockState;              /* SQLITE_SHM_xxxx constant showing lock state */
   u8 readerType;             /* SQLITE_SHM_READ or SQLITE_SHM_READ_FULL */
@@ -368,7 +368,8 @@ static int walIndexEntry(u32 iFrame){
 }
 
 /*
-** Release our reference to the wal-index memory map.
+** Release our reference to the wal-index memory map, if we are holding
+** it.
 */
 static void walIndexUnmap(Wal *pWal){
   if( pWal->pWiData ){
@@ -378,26 +379,42 @@ static void walIndexUnmap(Wal *pWal){
 }
 
 /*
-** Resize the wal-index file.  If newSize is negative, leave the size
+** Map the wal-index file into memory if it isn't already. 
-** unchanged.
+**
+** The reqSize parameter is the minimum required size of the mapping.
+** A value of -1 means "don't care".  The reqSize parameter is ignored
+** if the mapping is already held.
 */
-static int walIndexRemap(Wal *pWal, int newSize){
+static int walIndexMap(Wal *pWal, int reqSize){
-  int rc;
+  int rc = SQLITE_OK;
-  walIndexUnmap(pWal);
+  if( pWal->pWiData==0 ){
-  rc = pWal->pVfs->xShmSize(pWal->pWIndex, newSize,
+    rc = pWal->pVfs->xShmGet(pWal->pWIndex, reqSize, &pWal->szWIndex,
-                            &pWal->szWIndex, (void**)(char*)&pWal->pWiData);
+                             (void**)(char*)&pWal->pWiData);
     if( rc==SQLITE_OK && pWal->pWiData==0 ){
+      /* Make sure pWal->pWiData is not NULL while we are holding the
+      ** lock on the mapping. */
       assert( pWal->szWIndex==0 );
       pWal->pWiData = &pWal->iCallback;
     }
+  }
   return rc;
 }
 
 /*
-** Map the wal-index file into memory if it isn't already.
+** Remap the wal-index so that the mapping covers the full size
+** of the underlying file.
+**
+** If enlargeTo is non-negative, then increase the size of the underlying
+** storage to be at least as big as enlargeTo before remapping.
 */
-static int walIndexMap(Wal *pWal){
+static int walIndexRemap(Wal *pWal, int enlargeTo){
-  int rc = walIndexRemap(pWal, -1);
+  int rc;
+  int sz;
+  rc = pWal->pVfs->xShmSize(pWal->pWIndex, enlargeTo, &sz);
+  if( rc==SQLITE_OK && sz>pWal->szWIndex ){
+    walIndexUnmap(pWal);
+    rc = walIndexMap(pWal, sz);
+  }
   return rc;
 }
 
@@ -416,12 +433,12 @@ static int walIndexMap(Wal *pWal){
 static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
   u32 iSlot = walIndexEntry(iFrame);
   
-  walIndexMap(pWal);
+  walIndexMap(pWal, -1);
   while( (iSlot+128)>=pWal->szWIndex ){
     int rc;
     int nByte = pWal->szWIndex*4 + WALINDEX_MMAP_INCREMENT;
 
-    /* Unmap and remap the wal-index file. */
+    /* Enlarge the storage, then remap it. */
     rc = walIndexRemap(pWal, nByte);
     if( rc!=SQLITE_OK ){
       return rc;
@@ -640,7 +657,7 @@ static WalIterator *walIteratorInit(Wal *pWal){
   struct WalSegment *pFinal;      /* Final (unindexed) segment */
   u8 *aTmp;                       /* Temp space used by merge-sort */
 
-  walIndexMap(pWal);
+  walIndexMap(pWal, -1);
   aData = pWal->pWiData;
   iLast = pWal->hdr.iLastPg;
   nSegment = (iLast >> 8) + 1;
@@ -789,7 +806,8 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){
   u32 aHdr[WALINDEX_HDR_NFIELD+2];
 
   if( pWal->szWIndex==0 ){
-    int rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT);
+    int rc;
+    rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT);
     if( rc ) return rc;
   }
 
@@ -830,7 +848,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   int rc;
 
   assert( pWal->lockState>=SQLITE_SHM_READ );
-  walIndexMap(pWal);
+  walIndexMap(pWal, -1);
 
   /* First try to read the header without a lock. Verify the checksum
   ** before returning. This will almost always work.  
@@ -884,8 +902,7 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){
       if( pWal->hdr.iLastPg 
        && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex
       ){
-        rc = walIndexRemap(pWal, 0);
+         walIndexRemap(pWal, -1);
-        assert( rc || walIndexEntry(pWal->hdr.iLastPg)<pWal->szWIndex );
       }
     }
   }
@@ -913,7 +930,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){
   int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00);
 
   assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE );
-  walIndexMap(pWal);
+  walIndexMap(pWal, -1);
 
   /* Do a linear search of the unindexed block of page-numbers (if any) 
   ** at the end of the wal-index. An alternative to this would be to