When possible, use memory mapping when appending new pages to a database file.

FossilOrigin-Name: 14135da3cdbafd699563a29608f32347cda28338

manifest

@@ -1,5 +1,5 @@
-C Add\stest\sfile\smmap1.test.
-D 2013-03-20T10:07:43.411
+C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile.
+D 2013-03-20T14:26:59.370
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553
 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc
 F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0
 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30
-F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79
-F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48
+F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528
+F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b
 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
-F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f
+F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a
 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6
-F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc
-F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110
+F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150
+F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c
 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222
@@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9
 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca
-F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79
+F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75
 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796
@@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1
-R 1501a2a578b96ab1d025c2aefaaa8f35
+P aee1f53a74e636776cbbc11bdd5516432ad50533
+R 041b09b1e247c8054516280d9526a179
 U dan
-Z a7aa76ce5b116a8f55025f0891a0dfbb
+Z 8c309d7c56b08fcba504ca346091b196

manifest.uuid

@@ -1 +1 @@
-aee1f53a74e636776cbbc11bdd5516432ad50533
+14135da3cdbafd699563a29608f32347cda28338

src/os.c

@@ -140,8 +140,15 @@ int sqlite3OsShmMap(
   DO_OS_MALLOC_TEST(id);
   return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp);
 }
-int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){
-  return id->pMethods->xMremap(id, iOff, nOld, nNew, pp);
+int sqlite3OsMremap(
+  sqlite3_file *id,               /* Database file handle */
+  int flags,                      /* SQLITE_MREMAP_XXX flags */
+  i64 iOff,                       /* Offset at which mapping(s) start */
+  i64 nOld,                       /* Size of old mapping */
+  i64 nNew,                       /* Size of requested mapping */
+  void **pp                       /* IN/OUT: Pointer to mapped region */
+){
+  return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp);
 }
 
 /*

src/os.h

@@ -259,7 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
 int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
 void sqlite3OsShmBarrier(sqlite3_file *id);
 int sqlite3OsShmUnmap(sqlite3_file *id, int);
-int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **);
+int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **);
 
 
 /* 

src/os_unix.c

@@ -4434,6 +4434,7 @@ static int unixShmUnmap(
 */
 static int unixMremap(
   sqlite3_file *fd,               /* Main database file */
+  int flags,                      /* Mask of SQLITE_MREMAP_XXX flags */
   sqlite3_int64 iOff,             /* Offset to start mapping at */
   sqlite3_int64 nOld,             /* Size of old mapping, or zero */
   sqlite3_int64 nNew,             /* Size of new mapping, or zero */
@@ -4442,8 +4443,35 @@ static int unixMremap(
   unixFile *p = (unixFile *)fd;   /* The underlying database file */
   int rc = SQLITE_OK;             /* Return code */
   void *pNew = 0;                 /* New mapping */
+  i64 nRnd;                       /* nNew rounded up to 4096 */
 
   assert( iOff==0 );
+  nRnd = (nNew+4095) & ~(i64)((1 << 12)-1);
+
+  /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested 
+  ** mapping (nNew bytes) may be greater than the size of the database file.
+  ** If this is the case, extend the file on disk using ftruncate().  */
+  assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 );
+  if( flags & SQLITE_MREMAP_EXTEND ){
+    struct stat statbuf;          /* Low-level file information */
+    rc = osFstat(p->h, &statbuf);
+    if( rc==SQLITE_OK && nNew>statbuf.st_size ){
+      rc = robust_ftruncate(p->h, nNew);
+    }
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
+#if defined(_GNU_SOURCE) && defined(__linux__)
+  if( nRnd && nOld ){
+    void *pOld = *ppMap;
+    *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE);
+    if( pNew==MAP_FAILED ){
+      *ppMap = 0;
+      return SQLITE_IOERR_MREMAP;
+    }
+    return SQLITE_OK;
+  }
+#endif
 
   if( nOld!=0 ){
     void *pOld = *ppMap;
@@ -4453,11 +4481,10 @@ static int unixMremap(
   if( nNew>0 ){
     int flags = PROT_READ;
     if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;
-    nNew = (nNew+4095) & ~(i64)((1 << 12)-1);
-    pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff);
+    pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff);
     if( pNew==MAP_FAILED ){
       pNew = 0;
-      rc = SQLITE_IOERR;
+      rc = SQLITE_IOERR_MREMAP;
     }
   }
 

src/pager.c

@@ -3854,7 +3854,7 @@ static int pagerSyncHotJournal(Pager *pPager){
 */
 static int pagerUnmap(Pager *pPager){
   if( pPager->pMap ){
-    sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap);
+    sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap);
     pPager->nMap = 0;
     pPager->nMapValid = 0;
   }
@@ -3864,7 +3864,7 @@ static int pagerUnmap(Pager *pPager){
 /*
 ** Create, or recreate, the memory mapping of the database file.
 */
-static int pagerMap(Pager *pPager){
+static int pagerMap(Pager *pPager, int bExtend){
   int rc = SQLITE_OK;             /* Return code */
   Pgno nPg;                       /* Size of mapping to request in pages */
   i64 sz;                         /* Size of mapping to request in bytes */
@@ -3882,7 +3882,8 @@ static int pagerMap(Pager *pPager){
   if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit;
 
   if( sz!=pPager->nMapValid ){
-    rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap);
+    int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0);
+    rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap);
     if( rc==SQLITE_OK ){
       assert( pPager->pMap!=0 );
       pPager->nMap = sz;
@@ -4246,10 +4247,17 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){
   ** file size will be.
   */
   assert( rc!=SQLITE_OK || isOpen(pPager->fd) );
-  if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){
+  if( rc==SQLITE_OK 
+   && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize 
+  ){
     sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize;
     sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile);
     pPager->dbHintSize = pPager->dbSize;
+
+    if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){
+      pPager->dbFileSize = pPager->dbSize;
+      pagerMap(pPager, 1);
+    }
   }
 
   while( rc==SQLITE_OK && pList ){
@@ -5273,7 +5281,7 @@ int sqlite3PagerAcquire(
 
     if( bMmapOk ){
       if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){
-        rc = pagerMap(pPager);
+        rc = pagerMap(pPager, 0);
       }
       if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){
         if( pPager->eState>PAGER_READER ){

src/pager.h

@@ -108,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *);
 int sqlite3PagerSetPagesize(Pager*, u32*, int);
 int sqlite3PagerMaxPageCount(Pager*, int);
 void sqlite3PagerSetCachesize(Pager*, int);
+void sqlite3PagerSetMmapsize(Pager *, int);
 void sqlite3PagerShrink(Pager*);
 void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
 int sqlite3PagerLockingMode(Pager *, int);

src/sqlite.h.in

@@ -470,6 +470,7 @@ int sqlite3_exec(
 #define SQLITE_IOERR_SHMMAP            (SQLITE_IOERR | (21<<8))
 #define SQLITE_IOERR_SEEK              (SQLITE_IOERR | (22<<8))
 #define SQLITE_IOERR_DELETE_NOENT      (SQLITE_IOERR | (23<<8))
+#define SQLITE_IOERR_MREMAP            (SQLITE_IOERR | (24<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
 #define SQLITE_CANTOPEN_NOTEMPDIR      (SQLITE_CANTOPEN | (1<<8))
@@ -746,12 +747,14 @@ struct sqlite3_io_methods {
   void (*xShmBarrier)(sqlite3_file*);
   int (*xShmUnmap)(sqlite3_file*, int deleteFlag);
   /* Methods above are valid for version 2 */
-  int (*xMremap)(sqlite3_file *fd, 
+  int (*xMremap)(sqlite3_file *fd, int flags,
       sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap);
   /* Methods above are valid for version 3 */
   /* Additional methods may be added in future releases */
 };
 
+#define SQLITE_MREMAP_EXTEND  0x0001        /* xMremap call may extend file */
+
 /*
 ** CAPI3REF: Standard File Control Opcodes
 **