From b2d3de3bf47b7eaf5bb6bf68de5946895bd943af Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Mar 2013 18:34:37 +0000 Subject: [PATCH 01/72] Use mmap() to read from the database file in rollback mode. This branch is unix only for now. FossilOrigin-Name: 6f21d9cbf5d457e63a7282015a89ae785526cf6d --- manifest | 25 ++++---- manifest.uuid | 2 +- src/btree.c | 31 ++++++++++ src/os_unix.c | 4 ++ src/pager.c | 129 ++++++++++++++++++++++++++++++++++++++++- src/pcache.h | 2 + src/sqlite.h.in | 1 + test/permutations.test | 3 +- 8 files changed, 182 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 42eb32ea0f..a61f43717c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\stests\sfor\sticket\s[4dd95f6943]. -D 2013-03-13T07:02:04.083 +C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. +D 2013-03-14T18:34:37.796 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 746c4dafae8565b3be6fb9ce3bb1fa9f1e67cc59 +F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -160,13 +160,13 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f6387eef0cf8f6b808738f4f3aa47e6132af0940 +F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 582f8da52d0bd4b43d3bdaeba0ea7702c1f23702 +F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 -F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c +F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f2fa32f440dda59ca47e22889966b2a6eb3b491c +F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a @@ -656,7 +656,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 360b92859c0af814b3fe10b68746936389606501 +F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 839aa91faf1db7025d90fa3c65e50efb829b053b -R ad71abb6cd4a6ebc6857153126653bad +P 0b452734faa0839c817f040322e7733e423bfce2 +R d4c281f6a900c9b41816881ffb3c2331 +T *branch * experimental-mmap +T *sym-experimental-mmap * +T -sym-trunk * U dan -Z 69b39b8becc09fc4758d1b797bf5f4f2 +Z f16a594407ec9e95c26319dfbc751443 diff --git a/manifest.uuid b/manifest.uuid index 3d8be2e93e..dbbf6a71bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b452734faa0839c817f040322e7733e423bfce2 \ No newline at end of file +6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 07ec3fe52a..038126c3f0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2564,6 +2564,34 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } +/* +** If the shared-btree passed as the only argument is holding references +** to mmap pages, replace them with read/write pages. Return SQLITE_OK +** if successful, or an error code otherwise. +*/ +static int btreeSwapOutMmap(BtShared *pBt){ + BtCursor *pCsr; + for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ + int i; + for(i=0; i<=pCsr->iPage; i++){ + MemPage *pPg = pCsr->apPage[i]; + if( pPg->pDbPage->flags & PGHDR_MMAP ){ + int rc; + MemPage *pNew = 0; + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); + if( rc==SQLITE_OK && i==pCsr->iPage ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[i] = pNew; + releasePage(pPg); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + return SQLITE_OK; +} + /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2670,6 +2698,9 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = btreeSwapOutMmap(pBt); + } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } diff --git a/src/os_unix.c b/src/os_unix.c index 8f094bdc19..89326783d7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3623,6 +3623,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_GETFD: { + *(int*)pArg = pFile->h; + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and diff --git a/src/pager.c b/src/pager.c index 1d84fa2b7e..c362494093 100644 --- a/src/pager.c +++ b/src/pager.c @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + void *pMap; /* Memory mapped prefix of database file */ + i64 nMap; /* Size of mapping at pMap in bytes */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3803,6 +3808,105 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +#include + +/* +** Unmap any mapping of the database file. +*/ +static int pagerUnmap(Pager *pPager){ + if( pPager->pMap ){ + munmap(pPager->pMap, pPager->nMap); + pPager->pMap = 0; + pPager->nMap = 0; + } + return SQLITE_OK; +} + +static int pagerMap(Pager *pPager){ + int rc; + i64 sz = 0; + + assert( pPager->pMap==0 && pPager->nMap==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK && sz>0 ){ + int fd; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( rc==SQLITE_OK ){ + void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + if( pMap==MAP_FAILED ){ + assert( 0 ); + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMap = sz; + } + } + + return rc; +} + +static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ + int rc; + *ppPage = 0; + + assert( pPager->pWal==0 ); + + if( MEMDB==0 && pPager->tempFile==0 ){ + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + if( rc!=SQLITE_OK ) return rc; + } + + if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + PgHdr *p; + if( pPager->pFree ){ + p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ) return SQLITE_NOMEM; + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + *ppPage = p; + } + } + + return SQLITE_OK; +} + +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pFree; + pPager->pFree = pPg; +} + +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pFree; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -3823,6 +3927,8 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerUnmap(pPager); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4966,6 +5072,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + pagerUnmap(pPager); } } @@ -5007,7 +5114,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ pagerUnlockAndRollback(pPager); } } @@ -5083,6 +5190,15 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pPg ){ + *ppPage = pPg; + return SQLITE_OK; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5196,7 +5312,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5313,6 +5433,8 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + pagerUnmap(pPager); + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5535,6 +5657,9 @@ int sqlite3PagerWrite(DbPage *pDbPage){ assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); + /* There must not be any outstanding mmap pages at this point */ + assert( pPager->nMmapOut==0 ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pcache.h b/src/pcache.h index b9135fd859..f4d4ad71c1 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0373353d7b..0ddb0dd105 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -900,6 +900,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_GETFD 17 /* ** CAPI3REF: Mutex Handle diff --git a/test/permutations.test b/test/permutations.test index 711d4e57d3..d9014d77fe 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -135,7 +135,8 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { From 11dcd119136b3131f7520e1814232e333ce83d1e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 18:29:18 +0000 Subject: [PATCH 02/72] Allow read-only cursors to use mmap pages even if there is an open write transaction. FossilOrigin-Name: b387e2f9d24dccac1fd040e309f6fc7ec1cfffba --- manifest | 19 ++++------ manifest.uuid | 2 +- src/btree.c | 100 +++++++++++++++++++++++++++----------------------- src/pager.c | 73 ++++++++++++++++++++++++++---------- src/pager.h | 6 +++ 5 files changed, 124 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index a61f43717c..97b08ff77e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. -D 2013-03-14T18:34:37.796 +C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. +D 2013-03-15T18:29:18.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f +F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -162,8 +162,8 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a -F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 +F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1038,10 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0b452734faa0839c817f040322e7733e423bfce2 -R d4c281f6a900c9b41816881ffb3c2331 -T *branch * experimental-mmap -T *sym-experimental-mmap * -T -sym-trunk * +P 6f21d9cbf5d457e63a7282015a89ae785526cf6d +R 94aae53403ccd6ebc7d159fa0d23eb33 U dan -Z f16a594407ec9e95c26319dfbc751443 +Z 133f9be54c97c4ee392fb0706f346b90 diff --git a/manifest.uuid b/manifest.uuid index dbbf6a71bc..a20f3a4f02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file +b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 038126c3f0..5104625c35 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1569,13 +1569,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1618,9 +1622,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1628,7 +1633,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -2350,7 +2355,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2565,31 +2570,30 @@ int sqlite3BtreeNewDb(Btree *p){ } /* -** If the shared-btree passed as the only argument is holding references -** to mmap pages, replace them with read/write pages. Return SQLITE_OK -** if successful, or an error code otherwise. +** Ensure that any root page references held by open cursors are not +** mmap pages. */ static int btreeSwapOutMmap(BtShared *pBt){ - BtCursor *pCsr; - for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ - int i; - for(i=0; i<=pCsr->iPage; i++){ - MemPage *pPg = pCsr->apPage[i]; + int rc = SQLITE_OK; /* Return code */ + BtCursor *pCsr; /* Used to iterate through all open cursors */ + + for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ + if( pCsr->iPage>=0 ){ + MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ - int rc; MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && i==pCsr->iPage ){ + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } - pCsr->apPage[i] = pNew; + pCsr->apPage[0] = pNew; releasePage(pPg); if( rc!=SQLITE_OK ) return rc; } } } - return SQLITE_OK; + return rc; } /* @@ -2940,7 +2944,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3024,7 +3028,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3116,8 +3120,11 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( nOrig0 ){ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, nFin, nOrig, 0); + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); @@ -3438,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3872,7 +3879,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -4093,7 +4100,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4272,10 +4281,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4392,7 +4402,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -5006,7 +5016,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -5070,7 +5080,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -5150,7 +5160,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5198,7 +5208,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5212,7 +5222,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5280,7 +5290,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5307,7 +5317,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5353,7 +5363,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -6154,7 +6164,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -7245,7 +7255,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7266,7 +7276,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7342,7 +7352,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -7444,7 +7454,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7479,7 +7489,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7489,7 +7499,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7901,7 +7911,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; diff --git a/src/pager.c b/src/pager.c index c362494093..8470e7e35e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -658,6 +658,7 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ + i64 nMapValid; /* Bytes at pMap known to be valid */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* @@ -2512,6 +2513,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); + if( newSizenMapValid ){ + pPager->nMapValid = newSize; + } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -3818,6 +3822,7 @@ static int pagerUnmap(Pager *pPager){ munmap(pPager->pMap, pPager->nMap); pPager->pMap = 0; pPager->nMap = 0; + pPager->nMapValid = 0; } return SQLITE_OK; } @@ -3839,7 +3844,7 @@ static int pagerMap(Pager *pPager){ return SQLITE_IOERR; } pPager->pMap = pMap; - pPager->nMap = sz; + pPager->nMapValid = pPager->nMap = sz; } } @@ -3858,7 +3863,9 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ if( rc!=SQLITE_OK ) return rc; } - if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + if( pgno!=1 && pPager->pMap + && pPager->nMapValid>=((i64)pgno*pPager->pageSize) + ){ PgHdr *p; if( pPager->pFree ){ p = pPager->pFree; @@ -5035,9 +5042,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || pPager->pMap + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -5072,6 +5081,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + pagerUnmap(pPager); + }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ pagerUnmap(pPager); } } @@ -5173,10 +5191,20 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = ( + (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); @@ -5190,12 +5218,23 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pPg ){ - *ppPage = pPg; - return SQLITE_OK; + + if( bMmapOk ){ + if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + }else if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } } @@ -5433,8 +5472,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - pagerUnmap(pPager); - if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5653,13 +5690,11 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - /* There must not be any outstanding mmap pages at this point */ - assert( pPager->nMmapOut==0 ); - if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pager.h b/src/pager.h index 90f8e6af78..94957dac60 100644 --- a/src/pager.h +++ b/src/pager.h @@ -78,6 +78,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for From 7af6547a5afde7c5b48058d948df35857ba22a40 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 19:13:42 +0000 Subject: [PATCH 03/72] Fix a dropped error code in pager.c. FossilOrigin-Name: 022fdc986b33701abfd39621072ac3d9f9f7d43e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 97b08ff77e..ee60425958 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. -D 2013-03-15T18:29:18.146 +C Fix\sa\sdropped\serror\scode\sin\spager.c. +D 2013-03-15T19:13:42.183 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6f21d9cbf5d457e63a7282015a89ae785526cf6d -R 94aae53403ccd6ebc7d159fa0d23eb33 +P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba +R 53728d59c72e59e92889436c2f9b085b U dan -Z 133f9be54c97c4ee392fb0706f346b90 +Z 3a434ac42175412fea267939fd915765 diff --git a/manifest.uuid b/manifest.uuid index a20f3a4f02..9f9d488fb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file +022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8470e7e35e..23e7a67411 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5220,7 +5220,9 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); @@ -5232,10 +5234,11 @@ int sqlite3PagerAcquire( assert( rc==SQLITE_OK ); *ppPage = pPg; return SQLITE_OK; - }else if( rc!=SQLITE_OK ){ - goto pager_acquire_err; } } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); From a72014faf50889400af55631d4556a0adcffb593 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Mar 2013 20:19:21 +0000 Subject: [PATCH 04/72] When possible, use memcpy() to and from the mapped region instead of xWrite() and xRead(). FossilOrigin-Name: f8ca5622d99bedca957caa9ad311d798f63b3ce9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 41 +++++++++++++++++++++++++++++++++-------- src/test1.c | 24 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ee60425958..a6549b18c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdropped\serror\scode\sin\spager.c. -D 2013-03-15T19:13:42.183 +C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). +D 2013-03-16T20:19:21.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 +F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c ff3e68eedfbd858c9b89cf03e3db233cd29be1d0 +F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba -R 53728d59c72e59e92889436c2f9b085b +P 022fdc986b33701abfd39621072ac3d9f9f7d43e +R e9d1b00daf0ae79f0e154b8607fa8ad8 U dan -Z 3a434ac42175412fea267939fd915765 +Z 82853c015b40f8bb1665068f9eea77d4 diff --git a/manifest.uuid b/manifest.uuid index 9f9d488fb7..500dcc6707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file +f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 23e7a67411..d9b9ed5052 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2865,9 +2865,13 @@ static int readDbPage(PgHdr *pPg){ } if( rc==SQLITE_OK && !isInWal ){ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; + if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ + memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); + }else{ + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } } } @@ -3834,13 +3838,14 @@ static int pagerMap(Pager *pPager){ assert( pPager->pMap==0 && pPager->nMap==0 ); rc = sqlite3OsFileSize(pPager->fd, &sz); + sz = sz & ~(4096-1); + if( rc==SQLITE_OK && sz>0 ){ int fd; rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if( pMap==MAP_FAILED ){ - assert( 0 ); return SQLITE_IOERR; } pPager->pMap = pMap; @@ -3851,6 +3856,22 @@ static int pagerMap(Pager *pPager){ return rc; } +static int pagerRemap(Pager *pPager, Pgno nPage){ + i64 sz = (i64)nPage * pPager->pageSize; + sz = sz & ~(4096-1); + + if( pPager->nMap!=sz ){ + void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); + if( pMap==MAP_FAILED ){ + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMapValid = pPager->nMap = sz; + } + + return SQLITE_OK; +} + static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ int rc; *ppPage = 0; @@ -4225,7 +4246,11 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + if( pPager->nMapValid>=(offset+pPager->pageSize) ){ + memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); + }else{ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + } /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5089,8 +5114,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ pagerUnmap(pPager); - }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ - pagerUnmap(pPager); + }else if( pPager->pMap ){ + pagerRemap(pPager, nPage); } } diff --git a/src/test1.c b/src/test1.c index 0bace84420..a89b874a3b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,29 @@ static int test_test_control( return TCL_OK; } +#include +#include + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + r.ru_utime.tv_sec, r.ru_utime.tv_usec, + r.ru_stime.tv_sec, r.ru_stime.tv_usec, + r.ru_minflt, r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -6233,6 +6256,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, + { "getrusage", test_getrusage }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 5d8a137218d43070a2c635900d1ae7d7a7ef0f69 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Mar 2013 19:28:06 +0000 Subject: [PATCH 05/72] Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_size". FossilOrigin-Name: 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 --- manifest | 42 +++++----- manifest.uuid | 2 +- src/btree.c | 13 +++ src/btree.h | 1 + src/os.c | 3 + src/os.h | 1 + src/os_unix.c | 41 ++++++++- src/pager.c | 183 ++++++++++++++++++++++++----------------- src/pragma.c | 24 ++++++ src/sqlite.h.in | 21 +++++ src/sqliteInt.h | 1 + src/test1.c | 6 +- test/dbstatus2.test | 1 + test/exclusive2.test | 8 ++ test/func.test | 12 +-- test/incrblob.test | 4 + test/pageropt.test | 2 +- test/permutations.test | 8 ++ 18 files changed, 263 insertions(+), 110 deletions(-) diff --git a/manifest b/manifest index a6549b18c2..45c3ee6fc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). -D 2013-03-16T20:19:21.766 +C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". +D 2013-03-19T19:28:06.473 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a -F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd +F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -157,18 +157,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 +F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 +F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 +F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 +F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 +F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,15 +176,15 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e +F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 +F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test bf7396af964b89e39435babbcdf296ae8fc5f10a +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 372be98f6de44dd78734e364b7b626ea211761a6 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b058483c17952eff7797b837bbb61e27e6b05606 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 34765fa6fb5d8e0f256fc7d6497c04b205398849 +F test/incrblob.test bf210bea512474d4e1d94fbb9b0fcb386cd65dea F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -652,11 +652,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 +F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 +F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 022fdc986b33701abfd39621072ac3d9f9f7d43e -R e9d1b00daf0ae79f0e154b8607fa8ad8 +P f8ca5622d99bedca957caa9ad311d798f63b3ce9 +R 1add721bf7e9669d3b4d4c28d41bb13b U dan -Z 82853c015b40f8bb1665068f9eea77d4 +Z 3a3766130d03f0bab6c4936a9c26c7a5 diff --git a/manifest.uuid b/manifest.uuid index 500dcc6707..8d198c909c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file +6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5104625c35..2b7872fa8c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2130,6 +2130,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power diff --git a/src/btree.h b/src/btree.h index d4c9fe37d7..f2cca2fc71 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapSize(Btree*, int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os.c b/src/os.c index b5e918a727..45d8e0c121 100644 --- a/src/os.c +++ b/src/os.c @@ -140,6 +140,9 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ + return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +} /* ** The next group of routines are convenience wrappers around the diff --git a/src/os.h b/src/os.h index 1ec7d4ba11..76a60effdb 100644 --- a/src/os.h +++ b/src/os.h @@ -259,6 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index 89326783d7..f37f2404ee 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,42 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Map, remap or unmap part of the database file. +*/ +static int unixMremap( + sqlite3_file *fd, /* Main database file */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + unixFile *p = (unixFile *)fd; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + void *pNew = 0; /* New mapping */ + + assert( iOff==0 ); + + if( nOld!=0 ){ + void *pOld = *ppMap; + munmap(pOld, nOld); + } + + if( nNew>0 ){ + int flags = PROT_READ; + if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + nNew = (nNew+4095) & ~(i64)((1 << 12)-1); + pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + if( pNew==MAP_FAILED ){ + pNew = 0; + rc = SQLITE_IOERR; + } + } + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4487,7 +4523,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixMremap, /* xMremap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4504,7 +4541,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ diff --git a/src/pager.c b/src/pager.c index d9b9ed5052..9f7b349382 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,8 +659,11 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ i64 nMapValid; /* Bytes at pMap known to be valid */ + i64 nMapLimit; /* Maximum permitted mapping size */ + int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3356,6 +3359,35 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Set Pager.nMapLimit, the maximum permitted mapping size, based on the +** current values of Pager.nMapCfgLimit and Pager.pageSize. +** +** If this connection should not use mmap at all, set nMapLimit to zero. +*/ +static void pagerFixMaplimit(Pager *pPager){ + if( isOpen(pPager->fd)==0 + || pPager->fd->pMethods->iVersion<3 + || pPager->fd->pMethods->xMremap==0 + || pPager->tempFile + || pPager->pWal + ){ + pPager->nMapLimit = 0; + }else if( pPager->nMapCfgLimit<0 ){ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ + pPager->nMapCfgLimit = nMap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3591,6 +3623,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3816,108 +3849,100 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -#include - /* -** Unmap any mapping of the database file. +** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - munmap(pPager->pMap, pPager->nMap); - pPager->pMap = 0; + sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } return SQLITE_OK; } +/* +** Create, or recreate, the memory mapping of the database file. +*/ static int pagerMap(Pager *pPager){ - int rc; - i64 sz = 0; + int rc = SQLITE_OK; /* Return code */ + Pgno nPg; /* Size of mapping to request in pages */ + i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pMap==0 && pPager->nMap==0 ); + assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( pPager->pMap==0 || pPager->nMap>0 ); + assert( pPager->eState>=1 ); + assert( pPager->nMmapOut==0 ); + assert( pPager->nMapLimit>0 ); - rc = sqlite3OsFileSize(pPager->fd, &sz); - sz = sz & ~(4096-1); + /* Figure out how large a mapping to request. Set variable sz to this + ** value in bytes. */ + nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; + sz = (i64)nPg * pPager->pageSize; + if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - if( rc==SQLITE_OK && sz>0 ){ - int fd; - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( sz!=pPager->nMapValid ){ + rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; + assert( pPager->pMap!=0 ); + pPager->nMap = sz; + }else{ + assert( pPager->pMap==0 ); + pPager->nMap = 0; } + pPager->nMapValid = pPager->nMap; } + pPager->bMapResize = 0; return rc; } -static int pagerRemap(Pager *pPager, Pgno nPage){ - i64 sz = (i64)nPage * pPager->pageSize; - sz = sz & ~(4096-1); - - if( pPager->nMap!=sz ){ - void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; - } - - return SQLITE_OK; -} - +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The caller must ensure that page pgno lies within the currently mapped +** region. If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ - int rc; - *ppPage = 0; + PgHdr *p; /* Memory mapped page to return */ - assert( pPager->pWal==0 ); - - if( MEMDB==0 && pPager->tempFile==0 ){ - if( pPager->pMap==0 ){ - rc = pagerMap(pPager); - if( rc!=SQLITE_OK ) return rc; - } - - if( pgno!=1 && pPager->pMap - && pPager->nMapValid>=((i64)pgno*pPager->pageSize) - ){ - PgHdr *p; - if( pPager->pFree ){ - p = pPager->pFree; - pPager->pFree = p->pDirty; - p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); - }else{ - p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); - if( p==0 ) return SQLITE_NOMEM; - p->pExtra = (void *)&p[1]; - p->flags = PGHDR_MMAP; - p->nRef = 1; - p->pPager = pPager; - } - - assert( p->pExtra==(void *)&p[1] ); - assert( p->pPage==0 ); - assert( p->flags==PGHDR_MMAP ); - assert( p->pPager==pPager ); - assert( p->nRef==1 ); - - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; - p->pgno = pgno; - pPager->nMmapOut++; - *ppPage = p; + if( pPager->pFree ){ + *ppPage = p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + return SQLITE_NOMEM; } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; } + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + return SQLITE_OK; } +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; @@ -3925,6 +3950,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; } +/* +** Free all PgHdr objects stored in the Pager.pFree list. +*/ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; @@ -5115,7 +5143,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** possibility by unmapping the db here. */ pagerUnmap(pPager); }else if( pPager->pMap ){ - pagerRemap(pPager, nPage); + pPager->bMapResize = 1; } } @@ -5226,8 +5254,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = ( - (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5245,7 +5272,7 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ){ + if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ @@ -7014,6 +7041,7 @@ static int pagerOpenWal(Pager *pPager){ pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -7104,6 +7132,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/src/pragma.c b/src/pragma.c index f6dadbd4a8..9b31797d3f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -744,6 +744,30 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size + ** PRAGMA [database.]mmap_size=N + ** + ** Used to set or query the mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If it is set to a positive value, then it is + ** interpreted as a maximum size in pages. If set to less than zero, then + ** the absolute value is interpreted as a size limit in KB. + ** + ** The default value is zero (do not use memory mapped IO). + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->mmap_size = size; + sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0ddb0dd105..3fa30e01de 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -706,6 +706,24 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. +** +** Assuming parameter nNew is non-zero, the xMremap method should attempt +** to memory map a region nNew bytes in size starting at offset iOffset +** of the file. If successful, it should set *ppMap to point to the +** mapping and return SQLITE_OK. If the file is opened for read-write +** access, then the mapping should also be read-write. +** +** If nOld is non-zero, then the initial value of *ppMap points to a +** mapping returned by a previous call to xMremap. The existing mapping +** is nOld bytes in size and starts at offset iOffset of the file. In +** this case the xMremap method is expected to unmap the existing mapping +** and overwrite *ppMap with the pointer to the new mapping. If nOld is +** zero, then the initial value of *ppMap is undefined. +** +** If nNew is zero, then no new mapping should be created. Any old +** mapping must still be unmapped if nOld is non-zero. If the nOld +** parameter is non-zero, then the existing mapping is always unmapped - +** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -728,6 +746,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xMremap)(sqlite3_file *fd, + sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..59483bc87b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,6 +746,7 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ + int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/test1.c b/src/test1.c index a89b874a3b..500d60e0fe 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5859,9 +5859,9 @@ static int test_getrusage( getrusage(RUSAGE_SELF, &r); sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", - r.ru_utime.tv_sec, r.ru_utime.tv_usec, - r.ru_stime.tv_sec, r.ru_stime.tv_usec, - r.ru_minflt, r.ru_majflt + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt ); Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 18bb0870bd..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,6 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index 2208da5101..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} { return } +# Tests in this file verify that locking_mode=exclusive causes SQLite to +# use cached pages even if the database is changed on disk. This doesn't +# work with mmap. +if {[permutation]=="mmap"} { + finish_test + return +} + # This module does not work right if the cache spills at unexpected # moments. So disable the soft-heap-limit. # diff --git a/test/func.test b/test/func.test index f09ff49805..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,11 +1273,13 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x -} {many} +if {[permutation] != "mmap"} { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} +} do_test func-29.5 { db close sqlite3 db test.db diff --git a/test/incrblob.test b/test/incrblob.test index 7cc99dd983..f51f5852d4 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/pageropt.test b/test/pageropt.test index 82311965a5..de39bbaf05 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 $blobcontent] +} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index d9014d77fe..fd05b58e4f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -134,6 +134,14 @@ test_suite "veryquick" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + +test_suite "mmap" -prefix "mm-" -description { + Similar to veryquick. Except with memory mapping enabled. +} -presql { + pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ multiplex* server1.test shared2.test shared6.test From eecc3983741f501203faf8404eca56332418696e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 10:07:43 +0000 Subject: [PATCH 06/72] Add test file mmap1.test. FossilOrigin-Name: aee1f53a74e636776cbbc11bdd5516432ad50533 --- manifest | 11 ++++--- manifest.uuid | 2 +- test/mmap1.test | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 test/mmap1.test diff --git a/manifest b/manifest index 45c3ee6fc5..1a8ba2228d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". -D 2013-03-19T19:28:06.473 +C Add\stest\sfile\smmap1.test. +D 2013-03-20T10:07:43.411 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -630,6 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 +F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1038,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f8ca5622d99bedca957caa9ad311d798f63b3ce9 -R 1add721bf7e9669d3b4d4c28d41bb13b +P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 +R 1501a2a578b96ab1d025c2aefaaa8f35 U dan -Z 3a3766130d03f0bab6c4936a9c26c7a5 +Z a7aa76ce5b116a8f55025f0891a0dfbb diff --git a/manifest.uuid b/manifest.uuid index 8d198c909c..68ed5bb6a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file +aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test new file mode 100644 index 0000000000..343ad230b1 --- /dev/null +++ b/test/mmap1.test @@ -0,0 +1,84 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix mmap1 + +proc nRead {db} { + set bt [btree_from_db $db] + db_enter $db + array set stats [btree_pager_stats $bt] + db_leave $db + return $stats(read) +} + +foreach {t mmap_size nRead} { + 1 { PRAGMA mmap_size = -65536 } 4 + 2 { PRAGMA mmap_size = -50 } 156 + 3 { PRAGMA mmap_size = 0 } 344 +} { + do_multiclient_test tn { + sql1 $mmap_size + + code2 { + set ::rcnt 0 + proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n + } + db2 func rblob rblob + } + + sql2 { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + } + do_test $t.$tn.1 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 77} + + # Have connection 2 shrink the file. Check connection 1 can still read it. + sql2 { DELETE FROM t1 WHERE rowid%2; } + do_test $t.$tn.2 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {16 ok 42} + + # Have connection 2 grow the file. Check connection 1 can still read it. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.3 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 79} + + # Have connection 2 grow the file again. Check connection 1 is still ok. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.4 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {64 ok 149} + + # Check that the number of pages read by connection 1 indicates that the + # "PRAGMA mmap_size" command worked. + do_test $t.$tn.5 { nRead db } $nRead + } +} + + +finish_test + From eb97b29345592cf094b6639a985f9253653a5063 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 14:26:59 +0000 Subject: [PATCH 07/72] When possible, use memory mapping when appending new pages to a database file. FossilOrigin-Name: 14135da3cdbafd699563a29608f32347cda28338 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/os.c | 11 +++++++++-- src/os.h | 2 +- src/os_unix.c | 33 ++++++++++++++++++++++++++++++--- src/pager.c | 18 +++++++++++++----- src/pager.h | 1 + src/sqlite.h.in | 5 ++++- 8 files changed, 70 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 1a8ba2228d..d519ad3e2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\smmap1.test. -D 2013-03-20T10:07:43.411 +C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. +D 2013-03-20T14:26:59.370 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 -F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 +F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 +F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f +F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc -F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 +F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 +F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 -R 1501a2a578b96ab1d025c2aefaaa8f35 +P aee1f53a74e636776cbbc11bdd5516432ad50533 +R 041b09b1e247c8054516280d9526a179 U dan -Z a7aa76ce5b116a8f55025f0891a0dfbb +Z 8c309d7c56b08fcba504ca346091b196 diff --git a/manifest.uuid b/manifest.uuid index 68ed5bb6a8..f40916822a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file +14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 45d8e0c121..38757dcdb1 100644 --- a/src/os.c +++ b/src/os.c @@ -140,8 +140,15 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ - return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +int sqlite3OsMremap( + sqlite3_file *id, /* Database file handle */ + int flags, /* SQLITE_MREMAP_XXX flags */ + i64 iOff, /* Offset at which mapping(s) start */ + i64 nOld, /* Size of old mapping */ + i64 nNew, /* Size of requested mapping */ + void **pp /* IN/OUT: Pointer to mapped region */ +){ + return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); } /* diff --git a/src/os.h b/src/os.h index 76a60effdb..f08b92dcf0 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); +int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index f37f2404ee..1c26c6badf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,6 +4434,7 @@ static int unixShmUnmap( */ static int unixMremap( sqlite3_file *fd, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ @@ -4442,8 +4443,35 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ + i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); + nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(p->h, &statbuf); + if( rc==SQLITE_OK && nNew>statbuf.st_size ){ + rc = robust_ftruncate(p->h, nNew); + } + if( rc!=SQLITE_OK ) return rc; + } + +#if defined(_GNU_SOURCE) && defined(__linux__) + if( nRnd && nOld ){ + void *pOld = *ppMap; + *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + if( pNew==MAP_FAILED ){ + *ppMap = 0; + return SQLITE_IOERR_MREMAP; + } + return SQLITE_OK; + } +#endif if( nOld!=0 ){ void *pOld = *ppMap; @@ -4453,11 +4481,10 @@ static int unixMremap( if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - nNew = (nNew+4095) & ~(i64)((1 << 12)-1); - pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; - rc = SQLITE_IOERR; + rc = SQLITE_IOERR_MREMAP; } } diff --git a/src/pager.c b/src/pager.c index 9f7b349382..2848f9ad05 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3854,7 +3854,7 @@ static int pagerSyncHotJournal(Pager *pPager){ */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); + sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } @@ -3864,7 +3864,7 @@ static int pagerUnmap(Pager *pPager){ /* ** Create, or recreate, the memory mapping of the database file. */ -static int pagerMap(Pager *pPager){ +static int pagerMap(Pager *pPager, int bExtend){ int rc = SQLITE_OK; /* Return code */ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ @@ -3882,7 +3882,8 @@ static int pagerMap(Pager *pPager){ if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; if( sz!=pPager->nMapValid ){ - rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); + int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); + rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ assert( pPager->pMap!=0 ); pPager->nMap = sz; @@ -4246,10 +4247,17 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; + + if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ + pPager->dbFileSize = pPager->dbSize; + pagerMap(pPager, 1); + } } while( rc==SQLITE_OK && pList ){ @@ -5273,7 +5281,7 @@ int sqlite3PagerAcquire( if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager); + rc = pagerMap(pPager, 0); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ diff --git a/src/pager.h b/src/pager.h index 94957dac60..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapsize(Pager *, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3fa30e01de..c0a2803787 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,6 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -746,12 +747,14 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, + int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; +#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ + /* ** CAPI3REF: Standard File Control Opcodes ** From d306e1a3a113bd3905c6b661bcb8176b0bb6a844 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 18:25:49 +0000 Subject: [PATCH 08/72] Optimize the xMremap method in os_unix.c some. FossilOrigin-Name: 9529ed88a71fee02fae72dc86f0669bd6856ff92 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/os_unix.c | 61 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d519ad3e2a..c9eb696178 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. -D 2013-03-20T14:26:59.370 +C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. +D 2013-03-20T18:25:49.737 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a +F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aee1f53a74e636776cbbc11bdd5516432ad50533 -R 041b09b1e247c8054516280d9526a179 +P 14135da3cdbafd699563a29608f32347cda28338 +R ae7e97a190e1fdc49e8b167e558562e7 U dan -Z 8c309d7c56b08fcba504ca346091b196 +Z a86246ff854d247b1925a564cb8b08dc diff --git a/manifest.uuid b/manifest.uuid index f40916822a..3037557cd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file +9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1c26c6badf..7ab61ed0b3 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,16 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + /* ** Map, remap or unmap part of the database file. */ @@ -4443,10 +4453,10 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ - i64 nRnd; /* nNew rounded up to 4096 */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4461,10 +4471,43 @@ static int unixMremap( if( rc!=SQLITE_OK ) return rc; } + /* According to some sources, the effect of changing the size of the + ** underlying file on mapped regions that correspond to the added or + ** removed pages is undefined. However, there is reason to believe that + ** on modern platforms like Linux or OSX, things just work. For example, + ** it is possible to create a mapping larger than the file on disk and + ** extend the file on disk later on. + ** + ** Exploit this on OSX to reduce the number of munmap()/mmap() calls + ** if the file size is changing. In this case all mappings are rounded + ** up to the nearest 4MB. And if a new mapping is requested that has the + ** same rounded size as an old mapping, the old mapping can simply be + ** reused as is. + ** + ** It would be possible to do the above on Linux too. However, Linux has + ** the non-standard mremap() call to resize existing mappings, which can + ** be used instead. */ +#if defined(__APPLE__) + nNewRnd = ROUNDUP(nNew, 4096*1024); + nOldRnd = ROUNDUP(nOld, 4096*1024); +#else + nNewRnd = ROUNDUP(nNew, 4096*1); + nOldRnd = ROUNDUP(nOld, 4096*1); +#endif + + /* On OSX or Linux, reuse the old mapping if it is the right size. */ +#if defined(__APPLE__) || defined(__linux__) + if( nNewRnd==nOldRnd ){ + return SQLITE_OK; + } +#endif + + /* On Linux, if there is both an old and new mapping, resize the old + ** mapping using the non-standard mremap() call. */ #if defined(_GNU_SOURCE) && defined(__linux__) - if( nRnd && nOld ){ + if( nNewRnd && nOldRnd ){ void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); if( pNew==MAP_FAILED ){ *ppMap = 0; return SQLITE_IOERR_MREMAP; @@ -4473,15 +4516,17 @@ static int unixMremap( } #endif - if( nOld!=0 ){ + /* If we get this far, unmap any old mapping. */ + if( nOldRnd!=0 ){ void *pOld = *ppMap; - munmap(pOld, nOld); + munmap(pOld, nOldRnd); } - if( nNew>0 ){ + /* And, if required, use mmap() to create a new mapping. */ + if( nNewRnd>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; rc = SQLITE_IOERR_MREMAP; From c71b45e61944f6964b384614f3da737eff2c2dd6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 14:47:47 +0000 Subject: [PATCH 09/72] Do not use the Linux mremap() call. Use the same strategy for xMremap() as on OSX instead. FossilOrigin-Name: 5ed8ad780c991d2ca44003ee84350fb5e95ad58e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 30 ++++++------------------------ 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index c9eb696178..870e5cce8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. -D 2013-03-20T18:25:49.737 +C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. +D 2013-03-21T14:47:47.427 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 +F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 14135da3cdbafd699563a29608f32347cda28338 -R ae7e97a190e1fdc49e8b167e558562e7 +P 9529ed88a71fee02fae72dc86f0669bd6856ff92 +R 7e61f02cabbded666ed9e0992a636a37 U dan -Z a86246ff854d247b1925a564cb8b08dc +Z 67f89f7e2101ab4157bb24e7a7ac302a diff --git a/manifest.uuid b/manifest.uuid index 3037557cd4..f10e3bc724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file +5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7ab61ed0b3..153cf5f435 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4478,16 +4478,12 @@ static int unixMremap( ** it is possible to create a mapping larger than the file on disk and ** extend the file on disk later on. ** - ** Exploit this on OSX to reduce the number of munmap()/mmap() calls - ** if the file size is changing. In this case all mappings are rounded - ** up to the nearest 4MB. And if a new mapping is requested that has the - ** same rounded size as an old mapping, the old mapping can simply be - ** reused as is. - ** - ** It would be possible to do the above on Linux too. However, Linux has - ** the non-standard mremap() call to resize existing mappings, which can - ** be used instead. */ -#if defined(__APPLE__) + ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() + ** calls required if the file size is changing. In this case all mappings + ** are rounded up to the nearest 4MB. And if a new mapping is requested + ** that has the same rounded size as an old mapping, the old mapping can + ** be reused as is. */ +#if defined(__APPLE__) || defined(__linux__) nNewRnd = ROUNDUP(nNew, 4096*1024); nOldRnd = ROUNDUP(nOld, 4096*1024); #else @@ -4502,20 +4498,6 @@ static int unixMremap( } #endif - /* On Linux, if there is both an old and new mapping, resize the old - ** mapping using the non-standard mremap() call. */ -#if defined(_GNU_SOURCE) && defined(__linux__) - if( nNewRnd && nOldRnd ){ - void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); - if( pNew==MAP_FAILED ){ - *ppMap = 0; - return SQLITE_IOERR_MREMAP; - } - return SQLITE_OK; - } -#endif - /* If we get this far, unmap any old mapping. */ if( nOldRnd!=0 ){ void *pOld = *ppMap; From 5b068f863228ba1f074e2999a7dc1527e34d9612 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 15:57:32 +0000 Subject: [PATCH 10/72] Fix a problem when opening a write-transaction while there exist read-only b-tree cursors in mmap mode. FossilOrigin-Name: 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 870e5cce8d..8819c52a55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. -D 2013-03-21T14:47:47.427 +C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. +D 2013-03-21T15:57:32.562 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9529ed88a71fee02fae72dc86f0669bd6856ff92 -R 7e61f02cabbded666ed9e0992a636a37 +P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e +R fc9bd174ea58046e3b5bc29693aabeb6 U dan -Z 67f89f7e2101ab4157bb24e7a7ac302a +Z 811da101d49a049638aae5d61fad1590 diff --git a/manifest.uuid b/manifest.uuid index f10e3bc724..95a0f78145 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file +32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2b7872fa8c..0a51ff0301 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2595,7 +2595,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } From 7c49427fc25a0fc1738a4f7062abfea45642fbfc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:00:07 +0000 Subject: [PATCH 11/72] Avoid calling xRead() on a part of the database file that is memory mapped. FossilOrigin-Name: c8eac290a7240d69494bd0dad5ed1fdc2505f703 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8819c52a55..b91c073e14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. -D 2013-03-21T15:57:32.562 +C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. +D 2013-03-21T20:00:07.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e -R fc9bd174ea58046e3b5bc29693aabeb6 +P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc +R 78d8196d81dc00ab1ac5a50caf4d157f U dan -Z 811da101d49a049638aae5d61fad1590 +Z e76f59323062e94934c6de1e92e548e8 diff --git a/manifest.uuid b/manifest.uuid index 95a0f78145..0a2c6d8ff4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file +c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2848f9ad05..913ae74c46 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5130,9 +5130,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 ){ + if( nPage>0 || pPager->pMap ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( pPager->pMap ){ + memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); + }else{ + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + } if( rc!=SQLITE_OK ){ goto failed; } From b26e6c14fe6b89b89c0281319743e488d8bfe6ec Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:39:55 +0000 Subject: [PATCH 12/72] Fix cases where xRead() was being used to read from a memory mapped part of the database file. FossilOrigin-Name: 5c9e9df27b9f2c46cd55388a858d4e78ee564975 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 29 +++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index b91c073e14..4b2a8a6d5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. -D 2013-03-21T20:00:07.703 +C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. +D 2013-03-21T20:39:55.225 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 +F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc -R 78d8196d81dc00ab1ac5a50caf4d157f +P c8eac290a7240d69494bd0dad5ed1fdc2505f703 +R 0cdb7517bc23cd336050741c62097e56 U dan -Z e76f59323062e94934c6de1e92e548e8 +Z 0dd43854f08b06a022f11b1a174cdf13 diff --git a/manifest.uuid b/manifest.uuid index 0a2c6d8ff4..50136e7e1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file +5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 913ae74c46..0ff894bd87 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2087,6 +2087,24 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +/* +** Write nBuf bytes of data from buffer pBuf to offset iOff of the +** database file. If this part of the database file is memory mapped, +** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ + int rc = SQLITE_OK; + if( pPager->nMapValid>=(iOff+nBuf) ){ + memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); + }else{ + rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); + } + return rc; +} + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2261,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -3853,6 +3871,7 @@ static int pagerSyncHotJournal(Pager *pPager){ ** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ + assert( pPager->nMmapOut==0 ); if( pPager->pMap ){ sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; @@ -4282,11 +4301,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - if( pPager->nMapValid>=(offset+pPager->pageSize) ){ - memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); - }else{ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); - } + pagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -7086,6 +7101,8 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + pagerUnmap(pPager); + if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; From 6101d50471a97423032fbffcb72f5b62a28a90ee Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 08:58:38 +0000 Subject: [PATCH 13/72] Add assert statements to os_unix.c to ensure that any mapped region of the database file is not being read or written using the xRead() or xWrite() methods. FossilOrigin-Name: 765615f9fba7c1765eb741cb98a09a28b464ee55 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4b2a8a6d5d..7a0b83d9c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. -D 2013-03-21T20:39:55.225 +C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. +D 2013-03-22T08:58:38.319 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 +F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8eac290a7240d69494bd0dad5ed1fdc2505f703 -R 0cdb7517bc23cd336050741c62097e56 +P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 +R eb2eb83f96de836951dbdb14158ccee8 U dan -Z 0dd43854f08b06a022f11b1a174cdf13 +Z 42c0bd3cd106c51e7465866daa07e0a3 diff --git a/manifest.uuid b/manifest.uuid index 50136e7e1b..e0d24b066f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file +765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 153cf5f435..88f3177021 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,6 +249,8 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ + void *pMapRegion; /* Area memory mapped */ #endif #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3072,6 +3074,7 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3154,6 +3157,7 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -4457,6 +4461,8 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); + assert( p->mmapSize==nOld ); + assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4494,6 +4500,7 @@ static int unixMremap( /* On OSX or Linux, reuse the old mapping if it is the right size. */ #if defined(__APPLE__) || defined(__linux__) if( nNewRnd==nOldRnd ){ + VVA_ONLY( p->mmapSize = nNew; ) return SQLITE_OK; } #endif @@ -4502,6 +4509,7 @@ static int unixMremap( if( nOldRnd!=0 ){ void *pOld = *ppMap; munmap(pOld, nOldRnd); + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); } /* And, if required, use mmap() to create a new mapping. */ @@ -4511,7 +4519,10 @@ static int unixMremap( pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) rc = SQLITE_IOERR_MREMAP; + }else{ + VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) } } @@ -4846,6 +4857,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; From c00033125d612de5877ea2682cd255dd3199c00c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 17:46:11 +0000 Subject: [PATCH 14/72] Add a fix for the assert() statements added by the previous commit. FossilOrigin-Name: 19345416ed5e1ab5b0b35993b0b9069c2fb1683b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7a0b83d9c2..1be3bebe19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. -D 2013-03-22T08:58:38.319 +C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. +D 2013-03-22T17:46:11.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 +F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 -R eb2eb83f96de836951dbdb14158ccee8 +P 765615f9fba7c1765eb741cb98a09a28b464ee55 +R 125e508e18383c5f0a5b411a39a78cab U dan -Z 42c0bd3cd106c51e7465866daa07e0a3 +Z 4d1b2a2cebcc3fe5bdd2b5af2205540b diff --git a/manifest.uuid b/manifest.uuid index e0d24b066f..8efe087450 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file +19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 88f3177021..2f03c39670 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3470,6 +3470,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } + + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } #endif return SQLITE_OK; @@ -4461,7 +4469,7 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - assert( p->mmapSize==nOld ); + /* assert( p->mmapSize==nOld ); */ assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested From 99bd10979abd01877d6c213235c0ad22ab142a8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 18:20:14 +0000 Subject: [PATCH 15/72] Allow the database file to be memory mapped in wal mode. FossilOrigin-Name: d190ddabc386bc9654b99e33fb81b2f6e67b54d6 --- manifest | 18 +++++------ manifest.uuid | 2 +- src/pager.c | 88 +++++++++++++++++++++++++++++++++++++++++---------- src/pager.h | 4 +++ src/wal.c | 67 ++++++++++++++++++++------------------- src/wal.h | 7 ++-- 6 files changed, 124 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index 1be3bebe19..cab9a67764 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. -D 2013-03-22T17:46:11.890 +C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. +D 2013-03-22T18:20:14.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,8 +162,8 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 -F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 +F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 765615f9fba7c1765eb741cb98a09a28b464ee55 -R 125e508e18383c5f0a5b411a39a78cab +P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b +R 7243a86b7c03fc5d49450319f9d9b1bf U dan -Z 4d1b2a2cebcc3fe5bdd2b5af2205540b +Z 7b1a30ba76756800b3f32db2afc843bf diff --git a/manifest.uuid b/manifest.uuid index 8efe087450..4f8ecfff16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file +d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0ff894bd87..660514d508 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2095,7 +2095,7 @@ static void pagerReportSize(Pager *pPager){ ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ -static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ int rc = SQLITE_OK; if( pPager->nMapValid>=(iOff+nBuf) ){ memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); @@ -2279,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2864,11 +2864,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2880,11 +2879,10 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); @@ -2972,7 +2970,13 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + } + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3388,7 +3392,6 @@ static void pagerFixMaplimit(Pager *pPager){ || pPager->fd->pMethods->iVersion<3 || pPager->fd->pMethods->xMremap==0 || pPager->tempFile - || pPager->pWal ){ pPager->nMapLimit = 0; }else if( pPager->nMapCfgLimit<0 ){ @@ -3888,9 +3891,9 @@ static int pagerMap(Pager *pPager, int bExtend){ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( isOpen(pPager->fd) && pPager->tempFile==0 ); assert( pPager->pMap==0 || pPager->nMap>0 ); - assert( pPager->eState>=1 ); + /* assert( pPager->eState>=1 ); */ assert( pPager->nMmapOut==0 ); assert( pPager->nMapLimit>0 ); @@ -4213,6 +4216,46 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } +/* +** This is called by the wal.c module at the start of a checkpoint. If the +** checkpoint runs to completion, it will set the database file size to +** szReq bytes. This function performs two tasks: +** +** * If the file is currently less than szReq bytes in size, an +** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS +** layer of the expected file size, and +** +** * If mmap is being used, then the mapping is extended to szReq +** bytes in size. +** +** SQLITE_OK is returned if successful, or an error code if an error occurs. +*/ +int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ + int rc; + sqlite3_int64 sz; + + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->nMmapOut==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK ){ + if( sz>szReq ){ + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); + } + } + + if( rc==SQLITE_OK + && pPager->nMapLimit>0 + && pPager->nMapValidnMapValidnMapLimit + ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } + + return rc; +} + /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4275,7 +4318,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ pPager->dbFileSize = pPager->dbSize; - pagerMap(pPager, 1); + rc = pagerMap(pPager, 1); } } @@ -4301,7 +4344,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - pagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5275,6 +5318,7 @@ int sqlite3PagerAcquire( ){ int rc = SQLITE_OK; PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); /* It is acceptable to use a read-only (mmap) page for any page except @@ -5287,6 +5331,7 @@ int sqlite3PagerAcquire( assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5298,7 +5343,12 @@ int sqlite3PagerAcquire( rc = pPager->errCode; }else{ - if( bMmapOk ){ + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager, 0); } @@ -5378,9 +5428,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -7063,7 +7117,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); diff --git a/src/pager.h b/src/pager.h index 81ab30c115..970b5035bf 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,6 +174,10 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +int sqlite3PagerSetFilesize(Pager *, i64); + +/* Write data to the database file */ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); diff --git a/src/wal.c b/src/wal.c index 0d7271bf04..8757003d78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,6 +412,7 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ + Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1251,6 +1252,7 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1291,6 +1293,7 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; + pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1727,10 +1730,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ @@ -1744,7 +1744,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -2287,19 +2287,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2313,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2384,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). diff --git a/src/wal.h b/src/wal.h index a848de1527..96d4ff9adf 100644 --- a/src/wal.h +++ b/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -54,7 +53,8 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen( + sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); From aa1d67b1d92564e8360a07ff17c933d513542661 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 19:17:45 +0000 Subject: [PATCH 16/72] Fix a case where the mapped part of a database file may be written during a backup operation. FossilOrigin-Name: c8d67aefff7299dd5c8abeb2a3a52109c0a823ee --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/backup.c | 2 +- test/permutations.test | 2 +- test/speed1p.test | 3 +++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cab9a67764..cb648b797f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. -D 2013-03-22T18:20:14.143 +C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. +D 2013-03-22T19:17:45.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e +F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b -R 7243a86b7c03fc5d49450319f9d9b1bf +P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 +R d5a852613692dd4e943f0ee647615880 U dan -Z 7b1a30ba76756800b3f32db2afc843bf +Z db771a6d89978ace4423b73d7feafccd diff --git a/manifest.uuid b/manifest.uuid index 4f8ecfff16..caccf2eff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file +c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..8939bc67bc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); + rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/test/permutations.test b/test/permutations.test index fd05b58e4f..1015066e84 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -144,7 +144,7 @@ test_suite "mmap" -prefix "mm-" -description { pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..33be553b00 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,9 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=65536; + PRAGMA journal_mode=wal; + PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 23f29ec6284aaa01efdc2abfda257b10139670c8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Mar 2013 19:56:27 +0000 Subject: [PATCH 17/72] Fix a potential NULL-pointer dereference in btreeSwapOutMmap(). FossilOrigin-Name: e81ccdcd87a1387f7ca08d4d7c899311e2180d32 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cb648b797f..0a2be06535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. -D 2013-03-22T19:17:45.777 +C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). +D 2013-03-22T19:56:27.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee +F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 -R d5a852613692dd4e943f0ee647615880 -U dan -Z db771a6d89978ace4423b73d7feafccd +P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee +R dc31f859ef6cd2199f40f19f84f2b02e +U drh +Z 7f958188506961f9abc2573e8dff7118 diff --git a/manifest.uuid b/manifest.uuid index caccf2eff1..6c200497cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file +e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0a51ff0301..6a9761e91a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2593,7 +2593,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ if( pCsr->iPage>=0 ){ MemPage *pPg = pCsr->apPage[0]; - if( pPg->pDbPage->flags & PGHDR_MMAP ){ + if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ From 7909e54a22a0b466e5990d63a1c01de567deeb8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 20:15:31 +0000 Subject: [PATCH 18/72] Update wal mode tests so that they work with the mmap test permutation. FossilOrigin-Name: f7295872a36539d10edaf0c633f935f25cf73657 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/permutations.test | 3 +-- test/wal.test | 8 +++++--- test/wal5.test | 10 +++++++++- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 0a2be06535..868c9e0173 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). -D 2013-03-22T19:56:27.789 +C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. +D 2013-03-22T20:15:31.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc +F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -950,11 +950,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test a040047d7f2b9f34bc4d597964e5e7c09609c635 +F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 +F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee -R dc31f859ef6cd2199f40f19f84f2b02e -U drh -Z 7f958188506961f9abc2573e8dff7118 +P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 +R f5bd57e0d254253276d114325fd2ea6d +U dan +Z 0c2e4f926a5fed4f3d574bc2f3c43e20 diff --git a/manifest.uuid b/manifest.uuid index 6c200497cb..a8b241dcad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file +f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 1015066e84..6e6b5c90f7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,8 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] test_suite "valgrind" -prefix "" -description { diff --git a/test/wal.test b/test/wal.test index 24ce5f86a3..ca7edef952 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,6 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db +set nWal 37 +if {[permutation]=="mmap"} {set nWal 39} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; @@ -735,7 +737,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -745,7 +747,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -755,7 +757,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] #------------------------------------------------------------------------- diff --git a/test/wal5.test b/test/wal5.test index 6eceed5e59..d8bf541a4b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,15 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 4 1 4} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3} - do_test 2.3.$tn.8 { file_page_counts } {1 4 2 4} + + # The checkpoint above only writes page 1 of the db file. The other + # page (page 2) is locked by the read-transaction opened by the + # [sql2] commmand above. So normally, the db is 1 page in size here. + # However, in mmap() mode, the db is pre-allocated to 2 pages at the + # start of the checkpoint, even though page 2 cannot be written. + set nDb 1 + if {[permutation]=="mmap"} {set nDb 2} + do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } # Check that checkpoints block on the correct locks. And respond correctly From daf9a5a402fdcd69fce93e84bc61b46dcf9803e2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 09:56:39 +0000 Subject: [PATCH 19/72] Preliminary changes to support the xMremap VFS method on Windows. FossilOrigin-Name: 75a85a1c6ac59f2d021c28fa2161afb1118deea4 --- manifest | 16 ++-- manifest.uuid | 2 +- src/os_win.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++--- src/test1.c | 4 + 4 files changed, 203 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 868c9e0173..b81d7d1140 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. -D 2013-03-22T20:15:31.365 +C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. +D 2013-03-23T09:56:39.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 +F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e +F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 -R f5bd57e0d254253276d114325fd2ea6d -U dan -Z 0c2e4f926a5fed4f3d574bc2f3c43e20 +P f7295872a36539d10edaf0c633f935f25cf73657 +R 33ae184619389f0a2e2f45e36bcba41c +U mistachkin +Z 7a5b6171912da74b28162981ae14cd6f diff --git a/manifest.uuid b/manifest.uuid index a8b241dcad..ade491dce4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file +75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 970a94b5e8..fd0c8aeb89 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,11 +150,15 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -2061,6 +2065,9 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +/* Forward references to VFS methods */ +static int winUnmap(sqlite3_file *); + /* ** Close a file. ** @@ -2082,6 +2089,10 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2130,6 +2141,8 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); @@ -2176,6 +2189,7 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); @@ -2249,12 +2263,16 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested @@ -2267,11 +2285,20 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); + }else{ + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -3451,6 +3478,154 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +static int winUnmap( + sqlite3_file *id +){ + winFile *pFile = (winFile*)id; + assert( pFile!=0 ); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + return SQLITE_OK; +} + +/* +** Map, remap or unmap part of the database file. +*/ +static int winMremap( + sqlite3_file *id, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + winFile *pFile = (winFile*)id; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + HANDLE hMap = NULL; /* New mapping handle */ + void *pNew = 0; /* New mapping */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ + + assert( pFile!=0 ); + assert( iOff==0 ); + assert( nOld>=0 ); + assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); + assert( nNew>=0 ); + assert( ppMap ); + assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); + assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); + /* assert( pFile->mmapSize==nOld ); */ + + assert( winSysInfo.dwPageSize>0 ); + nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); + assert( nNewRnd>=0 ); + nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); + assert( nOldRnd>=0 ); + + if( nNewRnd==nOldRnd ){ + pFile->mmapSize = nNew; + return SQLITE_OK; + } + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + sqlite3_int64 oldSz; + rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK && nNewRnd>oldSz ){ + rc = winTruncate(id, nNewRnd); + } + if( rc!=SQLITE_OK ) return rc; + } + + /* If we get this far, unmap any old mapping. */ + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + + /* And, if required, create a new mapping. */ + if( nNewRnd>0 ){ + i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * + (winSysInfo.dwAllocationGranularity)); + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#endif + if( hMap==NULL ){ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap1", pFile->zPath); + return rc; + } + assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); + pNew = osMapViewOfFile(hMap, flags, + (DWORD)((offset>>32) & 0xffffffff), + (DWORD)(offset & 0xffffffff), + (SIZE_T)nNewRnd); +#endif + if( pNew==NULL ){ + osCloseHandle(hMap); + hMap = NULL; + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap2", pFile->zPath); + } + } + + pFile->hMap = hMap; + pFile->pMapRegion = pNew; + pFile->mmapSize = nNew; + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -3462,7 +3637,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3478,7 +3653,8 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winMremap, /* xMremap */ }; /**************************************************************************** @@ -3654,9 +3830,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3867,11 +4041,17 @@ static int winOpen( pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; OpenCounter(+1); return rc; @@ -4500,7 +4680,6 @@ int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4508,8 +4687,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; diff --git a/src/test1.c b/src/test1.c index 500d60e0fe..fb340f92d0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,7 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX #include #include @@ -5866,6 +5867,7 @@ static int test_getrusage( Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; } +#endif #if SQLITE_OS_WIN /* @@ -6256,7 +6258,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX { "getrusage", test_getrusage }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 46fce002d9ed09dc1db71678e73eb1336de96f5b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 10:09:46 +0000 Subject: [PATCH 20/72] Do not unmap the mapped file region in winTruncate as there may be outstanding pointers to it. FossilOrigin-Name: f57a9c91e993f76ce8b923e06e721414980e8e61 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b81d7d1140..a39a1c5070 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. -D 2013-03-23T09:56:39.403 +C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. +D 2013-03-23T10:09:46.701 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc +F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f7295872a36539d10edaf0c633f935f25cf73657 -R 33ae184619389f0a2e2f45e36bcba41c +P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 +R 2ac6e41fa14b2fcb69f496caa13f7e10 U mistachkin -Z 7a5b6171912da74b28162981ae14cd6f +Z 3096704ce1da72ac81b0db8561977d4e diff --git a/manifest.uuid b/manifest.uuid index ade491dce4..18b01de7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file +f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fd0c8aeb89..8d58ab0d44 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2270,9 +2270,6 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested From 2753388e8ae49d71e7b3edc71793950aa002c07d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 12:15:27 +0000 Subject: [PATCH 21/72] In the winMremap VFS function, unmap the region prior to attempting to truncate the file. FossilOrigin-Name: 8870c4cc6cda07154936644f7df25bd81d6ba38a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a39a1c5070..ada3b7db89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. -D 2013-03-23T10:09:46.701 +C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. +D 2013-03-23T12:15:27.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 +F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 -R 2ac6e41fa14b2fcb69f496caa13f7e10 +P f57a9c91e993f76ce8b923e06e721414980e8e61 +R a68c203c916fb468751938d0762a7812 U mistachkin -Z 3096704ce1da72ac81b0db8561977d4e +Z c2b92e0c0a8c400cb287f5f47ed82e78 diff --git a/manifest.uuid b/manifest.uuid index 18b01de7dd..0e0a52b9c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file +8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8d58ab0d44..8242a33849 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3560,7 +3560,10 @@ static int winMremap( sqlite3_int64 oldSz; rc = winFileSize(id, &oldSz); if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winTruncate(id, nNewRnd); + rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ + if( rc==SQLITE_OK ){ + rc = winTruncate(id, nNewRnd); + } } if( rc!=SQLITE_OK ) return rc; } From 9d56c6df9aef1e9cb59246b010c55407ea1ab73b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 14:20:42 +0000 Subject: [PATCH 22/72] Fix a case where a checkpoint operation could write to an invalid part of a memory mapped region. FossilOrigin-Name: 8dbe89d05ce91428c69003f0da79d883fa23e2b5 --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/pager.c | 16 ++++++------- test/mmap1.test | 57 +++++++++++++++++++++++++++++++++++++++++++---- test/speed1p.test | 3 --- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ada3b7db89..cf3732c267 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. -D 2013-03-23T12:15:27.020 +C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. +D 2013-03-23T14:20:42.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f +F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f57a9c91e993f76ce8b923e06e721414980e8e61 -R a68c203c916fb468751938d0762a7812 -U mistachkin -Z c2b92e0c0a8c400cb287f5f47ed82e78 +P 8870c4cc6cda07154936644f7df25bd81d6ba38a +R 0b90a49ac5025c5ac7d6b4eb49fe1878 +U dan +Z c9ca2a38d0bbfc316a52f1d1e135baf9 diff --git a/manifest.uuid b/manifest.uuid index 0e0a52b9c3..bd17b56cca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file +8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 660514d508..09cc4a2375 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4232,7 +4232,7 @@ static int syncJournal(Pager *pPager, int newHdr){ */ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ int rc; - sqlite3_int64 sz; + i64 sz; /* Size of file on disk in bytes */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->nMmapOut==0 ); @@ -4244,13 +4244,13 @@ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ } } - if( rc==SQLITE_OK - && pPager->nMapLimit>0 - && pPager->nMapValidnMapValidnMapLimit - ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); + + if( rc==SQLITE_OK ){ + i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; + if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } } return rc; diff --git a/test/mmap1.test b/test/mmap1.test index 343ad230b1..41714e29d8 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,13 +23,17 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead} { - 1 { PRAGMA mmap_size = -65536 } 4 - 2 { PRAGMA mmap_size = -50 } 156 - 3 { PRAGMA mmap_size = 0 } 344 +foreach {t mmap_size nRead c2init} { + 1.1 { PRAGMA mmap_size = -65536 } 4 {} + 1.2 { PRAGMA mmap_size = -50 } 156 {} + 1.3 { PRAGMA mmap_size = 0 } 344 {} + 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} + 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} } { do_multiclient_test tn { sql1 $mmap_size + sql2 $c2init code2 { set ::rcnt 0 @@ -79,6 +83,51 @@ foreach {t mmap_size nRead} { } } +set ::rcnt 0 +proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n +} + +reset_db +db func rblob rblob + +do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = -65536; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + PRAGMA wal_checkpoint; +} {wal 0 103 103} + +do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; +} {1 32} + +do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } +} {16} + +do_execsql_test 2.4 { + PRAGMA wal_checkpoint; +} {0 24 24} finish_test diff --git a/test/speed1p.test b/test/speed1p.test index 33be553b00..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,9 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=65536; - PRAGMA journal_mode=wal; - PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From e5586a1a79f3d8e61ee6d263ec680c012ffbabfb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 17:29:06 +0000 Subject: [PATCH 23/72] Improve a comment in wal.c. No code changes. FossilOrigin-Name: 60b9f5e4dd2af54975ba78437239f0bebd472fd2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cf3732c267..6660b0b860 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. -D 2013-03-23T14:20:42.975 +C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. +D 2013-03-23T17:29:06.036 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8870c4cc6cda07154936644f7df25bd81d6ba38a -R 0b90a49ac5025c5ac7d6b4eb49fe1878 +P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 +R b063a8588944a5f842812de15381707f U dan -Z c9ca2a38d0bbfc316a52f1d1e135baf9 +Z 350cbe536c1fd5aa73b476fafae6db5c diff --git a/manifest.uuid b/manifest.uuid index bd17b56cca..7aea0f37ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file +60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 8757003d78..051b77f515 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1725,12 +1725,17 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ + /* If the database file is currently smaller than mxPage pages in size, + ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to + ** inform it that it is likely to grow to that size. + ** + ** Additionally, if the pager is using mmap(), then the call to + ** SetFilesize() guarantees that the mapping is not larger than mxPage + ** pages. This makes the sqlite3OsTruncate() call below safe - no pages + ** that are part of the mapped region will be truncated away. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ From f23da96636f9de1a81c939bd7a0cc25ee1d37f49 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 21:00:41 +0000 Subject: [PATCH 24/72] Replace the sqlite3_io_methods.xMremap interface with sqlite3_io_methods.xFetch and xUnfetch. FossilOrigin-Name: 1431be95579160fb70408d43e17fc23c7b69ab4a --- manifest | 28 +++--- manifest.uuid | 2 +- src/backup.c | 2 +- src/os.c | 15 ++-- src/os.h | 3 +- src/os_unix.c | 211 +++++++++++++++++++++++++++------------------ src/pager.c | 221 ++++++++++++------------------------------------ src/pager.h | 3 - src/sqlite.h.in | 30 ++----- src/wal.c | 22 ++--- src/wal.h | 3 +- 11 files changed, 222 insertions(+), 318 deletions(-) diff --git a/manifest b/manifest index 6660b0b860..8d71dc88ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. -D 2013-03-23T17:29:06.036 +C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. +D 2013-03-23T21:00:41.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce +F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 -F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b +F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 +F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf +F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 -F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 +F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 +F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f +F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 -F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 +F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 -R b063a8588944a5f842812de15381707f +P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 +R eee4d88d6400cf65980c5b201532d2c9 U dan -Z 350cbe536c1fd5aa73b476fafae6db5c +Z eabd20cdc7aabbc65591f96e4f6dff04 diff --git a/manifest.uuid b/manifest.uuid index 7aea0f37ae..f378aa2e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file +1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 8939bc67bc..71a8a1a3e7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); + rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/src/os.c b/src/os.c index 38757dcdb1..08e0d42172 100644 --- a/src/os.c +++ b/src/os.c @@ -140,15 +140,12 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap( - sqlite3_file *id, /* Database file handle */ - int flags, /* SQLITE_MREMAP_XXX flags */ - i64 iOff, /* Offset at which mapping(s) start */ - i64 nOld, /* Size of old mapping */ - i64 nNew, /* Size of requested mapping */ - void **pp /* IN/OUT: Pointer to mapped region */ -){ - return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); + +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, void *p){ + return id->pMethods->xUnfetch(id, p); } /* diff --git a/src/os.h b/src/os.h index f08b92dcf0..a7dacfb6ea 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 2f03c39670..334719200a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,9 +249,14 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ - void *pMapRegion; /* Area memory mapped */ + #endif + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ + int nFetchOut; /* Number of outstanding xFetch refs */ + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -1805,6 +1810,9 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1817,6 +1825,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3074,7 +3083,6 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3085,6 +3093,21 @@ static int unixRead( ); #endif + /* Deal with as much of this write request as possible by transfering + ** data to the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3157,7 +3180,6 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3190,6 +3212,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3470,6 +3507,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } +#endif /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will @@ -3478,7 +3516,6 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nBytemmapSize ){ pFile->mmapSize = nByte; } -#endif return SQLITE_OK; } @@ -3568,6 +3605,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3635,8 +3685,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_GETFD: { - *(int*)pArg = pFile->h; + case SQLITE_FCNTL_MMAP_SIZE: { + pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } #ifdef SQLITE_DEBUG @@ -4451,91 +4501,86 @@ static int unixShmUnmap( */ #define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) -/* -** Map, remap or unmap part of the database file. -*/ -static int unixMremap( - sqlite3_file *fd, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - unixFile *p = (unixFile *)fd; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + munmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} - assert( iOff==0 ); - /* assert( p->mmapSize==nOld ); */ - assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(p->h, &statbuf); - if( rc==SQLITE_OK && nNew>statbuf.st_size ){ - rc = robust_ftruncate(p->h, nNew); + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; } - /* According to some sources, the effect of changing the size of the - ** underlying file on mapped regions that correspond to the added or - ** removed pages is undefined. However, there is reason to believe that - ** on modern platforms like Linux or OSX, things just work. For example, - ** it is possible to create a mapping larger than the file on disk and - ** extend the file on disk later on. - ** - ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() - ** calls required if the file size is changing. In this case all mappings - ** are rounded up to the nearest 4MB. And if a new mapping is requested - ** that has the same rounded size as an old mapping, the old mapping can - ** be reused as is. */ -#if defined(__APPLE__) || defined(__linux__) - nNewRnd = ROUNDUP(nNew, 4096*1024); - nOldRnd = ROUNDUP(nOld, 4096*1024); -#else - nNewRnd = ROUNDUP(nNew, 4096*1); - nOldRnd = ROUNDUP(nOld, 4096*1); -#endif + if( nMap!=pFd->mmapSize ){ + void *pNew; + unixUnmapfile(pFd); - /* On OSX or Linux, reuse the old mapping if it is the right size. */ -#if defined(__APPLE__) || defined(__linux__) - if( nNewRnd==nOldRnd ){ - VVA_ONLY( p->mmapSize = nNew; ) - return SQLITE_OK; - } -#endif + if( nMap>0 ){ + void *pNew; + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MREMAP; + } - /* If we get this far, unmap any old mapping. */ - if( nOldRnd!=0 ){ - void *pOld = *ppMap; - munmap(pOld, nOldRnd); - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); - } - - /* And, if required, use mmap() to create a new mapping. */ - if( nNewRnd>0 ){ - int flags = PROT_READ; - if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); - if( pNew==MAP_FAILED ){ - pNew = 0; - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) - rc = SQLITE_IOERR_MREMAP; - }else{ - VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) + pFd->pMapRegion = pNew; + pFd->mmapOrigsize = pFd->mmapSize = nMap; } } - *ppMap = pNew; - return rc; + return SQLITE_OK; +} + +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +static int unixUnfetch(sqlite3_file *fd, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + assert( (p==0)==(pFd->nFetchOut==0) ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -4597,7 +4642,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ - unixMremap, /* xMremap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4865,7 +4911,6 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index 09cc4a2375..7e00e1431a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -656,14 +656,10 @@ struct Pager { int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - void *pMap; /* Memory mapped prefix of database file */ - i64 nMap; /* Size of mapping at pMap in bytes */ - i64 nMapValid; /* Bytes at pMap known to be valid */ - i64 nMapLimit; /* Maximum permitted mapping size */ + u8 bUseFetch; /* True to use xFetch() */ int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ - int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2087,24 +2083,6 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif -/* -** Write nBuf bytes of data from buffer pBuf to offset iOff of the -** database file. If this part of the database file is memory mapped, -** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ - int rc = SQLITE_OK; - if( pPager->nMapValid>=(iOff+nBuf) ){ - memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); - }else{ - rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); - } - return rc; -} - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2279,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2534,9 +2512,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - if( newSizenMapValid ){ - pPager->nMapValid = newSize; - } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -2884,13 +2859,9 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ - memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); - }else{ - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; } } @@ -3120,6 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); } return rc; @@ -3382,22 +3354,24 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Set Pager.nMapLimit, the maximum permitted mapping size, based on the -** current values of Pager.nMapCfgLimit and Pager.pageSize. -** -** If this connection should not use mmap at all, set nMapLimit to zero. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. */ static void pagerFixMaplimit(Pager *pPager){ - if( isOpen(pPager->fd)==0 - || pPager->fd->pMethods->iVersion<3 - || pPager->fd->pMethods->xMremap==0 - || pPager->tempFile - ){ - pPager->nMapLimit = 0; - }else if( pPager->nMapCfgLimit<0 ){ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + if( pPager->bUseFetch ){ + void *p; + i64 nMapLimit; + if( pPager->nMapCfgLimit<0 ){ + nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } + + p = (void *)&nMapLimit; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + } } } @@ -3870,67 +3844,22 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -/* -** Unmap any memory mapping of the database file. -*/ -static int pagerUnmap(Pager *pPager){ - assert( pPager->nMmapOut==0 ); - if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); - pPager->nMap = 0; - pPager->nMapValid = 0; - } - return SQLITE_OK; -} - -/* -** Create, or recreate, the memory mapping of the database file. -*/ -static int pagerMap(Pager *pPager, int bExtend){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPg; /* Size of mapping to request in pages */ - i64 sz; /* Size of mapping to request in bytes */ - - assert( isOpen(pPager->fd) && pPager->tempFile==0 ); - assert( pPager->pMap==0 || pPager->nMap>0 ); - /* assert( pPager->eState>=1 ); */ - assert( pPager->nMmapOut==0 ); - assert( pPager->nMapLimit>0 ); - - /* Figure out how large a mapping to request. Set variable sz to this - ** value in bytes. */ - nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; - sz = (i64)nPg * pPager->pageSize; - if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - - if( sz!=pPager->nMapValid ){ - int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); - rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); - if( rc==SQLITE_OK ){ - assert( pPager->pMap!=0 ); - pPager->nMap = sz; - }else{ - assert( pPager->pMap==0 ); - pPager->nMap = 0; - } - pPager->nMapValid = pPager->nMap; - } - pPager->bMapResize = 0; - - return rc; -} - /* ** Obtain a reference to a memory mapped page object for page number pgno. -** The caller must ensure that page pgno lies within the currently mapped -** region. If successful, set *ppPage to point to the new page reference +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set ** *ppPage to zero. ** ** Page references obtained by calling this function should be released ** by calling pagerReleaseMapPage(). */ -static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ PgHdr *p; /* Memory mapped page to return */ if( pPager->pFree ){ @@ -3955,8 +3884,8 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ assert( p->pPager==pPager ); assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; p->pgno = pgno; + p->pData = pData; pPager->nMmapOut++; return SQLITE_OK; @@ -3971,6 +3900,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->nMmapOut--; pPg->pDirty = pPager->pFree; pPager->pFree = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, pPg->pData); } /* @@ -4006,7 +3938,6 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pagerUnmap(pPager); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; @@ -4216,46 +4147,6 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } -/* -** This is called by the wal.c module at the start of a checkpoint. If the -** checkpoint runs to completion, it will set the database file size to -** szReq bytes. This function performs two tasks: -** -** * If the file is currently less than szReq bytes in size, an -** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS -** layer of the expected file size, and -** -** * If mmap is being used, then the mapping is extended to szReq -** bytes in size. -** -** SQLITE_OK is returned if successful, or an error code if an error occurs. -*/ -int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ - int rc; - i64 sz; /* Size of file on disk in bytes */ - - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->nMmapOut==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - if( rc==SQLITE_OK ){ - if( sz>szReq ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); - } - } - - - if( rc==SQLITE_OK ){ - i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; - if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); - } - } - - return rc; -} - /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4315,11 +4206,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; - - if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ - pPager->dbFileSize = pPager->dbSize; - rc = pagerMap(pPager, 1); - } } while( rc==SQLITE_OK && pList ){ @@ -4344,7 +4230,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5164,7 +5050,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->pMap + || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5188,13 +5074,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 || pPager->pMap ){ + if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - if( pPager->pMap ){ - memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); - }else{ - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - } + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ goto failed; } @@ -5211,9 +5093,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - pagerUnmap(pPager); - }else if( pPager->pMap ){ - pPager->bMapResize = 1; + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0); + } } } @@ -5325,7 +5207,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 + const int bMmapOk = (pgno!=1 && pPager->bUseFetch && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5349,15 +5231,20 @@ int sqlite3PagerAcquire( } if( iFrame==0 && bMmapOk ){ - if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager, 0); - } - if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, pData); } if( pPg ){ assert( rc==SQLITE_OK ); @@ -7117,7 +7004,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); @@ -7155,8 +7042,6 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - pagerUnmap(pPager); - if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; diff --git a/src/pager.h b/src/pager.h index 970b5035bf..066d1f11d7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -176,9 +176,6 @@ int sqlite3SectorSize(sqlite3_file *); void sqlite3PagerTruncateImage(Pager*,Pgno); int sqlite3PagerSetFilesize(Pager *, i64); -/* Write data to the database file */ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); - #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c0a2803787..666a2e084d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -707,24 +707,6 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. -** -** Assuming parameter nNew is non-zero, the xMremap method should attempt -** to memory map a region nNew bytes in size starting at offset iOffset -** of the file. If successful, it should set *ppMap to point to the -** mapping and return SQLITE_OK. If the file is opened for read-write -** access, then the mapping should also be read-write. -** -** If nOld is non-zero, then the initial value of *ppMap points to a -** mapping returned by a previous call to xMremap. The existing mapping -** is nOld bytes in size and starts at offset iOffset of the file. In -** this case the xMremap method is expected to unmap the existing mapping -** and overwrite *ppMap with the pointer to the new mapping. If nOld is -** zero, then the initial value of *ppMap is undefined. -** -** If nNew is zero, then no new mapping should be created. Any old -** mapping must still be unmapped if nOld is non-zero. If the nOld -** parameter is non-zero, then the existing mapping is always unmapped - -** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -747,14 +729,12 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, int flags, - sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; -#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ - /* ** CAPI3REF: Standard File Control Opcodes ** @@ -906,6 +886,10 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The argument is assumed to point to a value of type sqlite3_int64. An +** advisory maximum amount of this file to memory map in bytes. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -924,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_GETFD 17 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/wal.c b/src/wal.c index 051b77f515..4602eb2c34 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,7 +412,6 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ - Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1252,7 +1251,6 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1293,7 +1291,6 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; - pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1725,19 +1722,18 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file is currently smaller than mxPage pages in size, - ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to - ** inform it that it is likely to grow to that size. - ** - ** Additionally, if the pager is using mmap(), then the call to - ** SetFilesize() guarantees that the mapping is not larger than mxPage - ** pages. This makes the sqlite3OsTruncate() call below safe - no pages - ** that are part of the mapped region will be truncated away. */ + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -1749,7 +1745,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } diff --git a/src/wal.h b/src/wal.h index 96d4ff9adf..ff7624af63 100644 --- a/src/wal.h +++ b/src/wal.h @@ -53,8 +53,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen( - sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ From a1afc7425a42b774b307416b3256d28c1d4a185f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 13:50:49 +0000 Subject: [PATCH 25/72] Fix a case in the pager where an xFetch() reference was being leaked following an OOM error. FossilOrigin-Name: 5885ba6ce768658ec25b60747430d147b315b55c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 1 + src/pager.h | 1 - 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8d71dc88ab..5e9f5ad099 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. -D 2013-03-23T21:00:41.457 +C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. +D 2013-03-25T13:50:49.946 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,10 +160,10 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 +F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 -F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 +F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 -R eee4d88d6400cf65980c5b201532d2c9 +P 1431be95579160fb70408d43e17fc23c7b69ab4a +R 2702651dc0367254a02a7905a85901b0 U dan -Z eabd20cdc7aabbc65591f96e4f6dff04 +Z 4827c382c08dac8144315d5de3ed2c37 diff --git a/manifest.uuid b/manifest.uuid index f378aa2e8b..37accb35b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file +5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 334719200a..d865810570 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1807,6 +1807,7 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } @@ -4531,7 +4532,6 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew; unixUnmapfile(pFd); if( nMap>0 ){ diff --git a/src/pager.c b/src/pager.c index 7e00e1431a..d6e6b5b440 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3870,6 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; diff --git a/src/pager.h b/src/pager.h index 066d1f11d7..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,7 +174,6 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -int sqlite3PagerSetFilesize(Pager *, i64); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 11ac84635b840afb91178063bd3fa30fff584963 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 14:31:39 +0000 Subject: [PATCH 26/72] Do not return SQLITE_IOERR when the user attempts to open a small file that is not a database with mmap enabled. Instead return SQLITE_NOTADB. FossilOrigin-Name: bbcaab3e80d0ff776d8567094b137d1483b3377b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5e9f5ad099..01d74c7724 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. -D 2013-03-25T13:50:49.946 +C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. +D 2013-03-25T14:31:39.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1431be95579160fb70408d43e17fc23c7b69ab4a -R 2702651dc0367254a02a7905a85901b0 +P 5885ba6ce768658ec25b60747430d147b315b55c +R c243df2223b3431445e14c1469df7b20 U dan -Z 4827c382c08dac8144315d5de3ed2c37 +Z 6b851eca8762089c75296a1160d8987e diff --git a/manifest.uuid b/manifest.uuid index 37accb35b1..b4cdb9d207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file +bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d6e6b5b440..927126c393 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5051,7 +5051,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ + || pPager->bUseFetch )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5078,7 +5078,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ From aef49d7141d9bbbc68a2c8ce7bd17cdafb06aa6b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 16:28:54 +0000 Subject: [PATCH 27/72] Remove unnecessary code to round the size of a memory mapping to 4KB from os_unix.c. Rename SQLITE_IOERR_MREMAP to SQLITE_IOERR_MMAP. Fix other small issues in os_unix.c. FossilOrigin-Name: dce35c01a5fe66d2970075b1e3f0376026485e4c --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 54 +++++++++++++++++++++++++++++++++++++++---------- src/sqlite.h.in | 2 +- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 01d74c7724..919b150ea9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. -D 2013-03-25T14:31:39.689 +C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. +D 2013-03-25T16:28:54.437 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 +F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 +F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5885ba6ce768658ec25b60747430d147b315b55c -R c243df2223b3431445e14c1469df7b20 +P bbcaab3e80d0ff776d8567094b137d1483b3377b +R fd25f18074b1cc787a38214a338c9127 U dan -Z 6b851eca8762089c75296a1160d8987e +Z ae05d407cd866087491e3302af269873 diff --git a/manifest.uuid b/manifest.uuid index b4cdb9d207..23ee7e3b08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file +dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d865810570..a97aba87f1 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4493,15 +4493,8 @@ static int unixShmUnmap( #endif /* #ifndef SQLITE_OMIT_WAL */ /* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 +** If it is currently memory mapped, unmap file pFd. */ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ @@ -4512,6 +4505,22 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ static int unixMapfile(unixFile *pFd, i64 nByte){ i64 nMap = nByte; int rc; @@ -4538,19 +4547,32 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MREMAP; + return SQLITE_IOERR_MMAP; } pFd->pMapRegion = pNew; - pFd->mmapOrigsize = pFd->mmapSize = nMap; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } } return SQLITE_OK; } +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ *pp = 0; @@ -4568,9 +4590,19 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return SQLITE_OK; } +/* +** If the second argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). Or, if the second +** argument is NULL, then this function is being called to inform the VFS +** layer that, according to POSIX, any existing mapping may now be invalid +** and should be unmapped. +*/ static int unixUnfetch(sqlite3_file *fd, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); if( p ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 666a2e084d..7bbd3fa000 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,7 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) -#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) From df737fe6f51522c478235ff6ddc478243f718bf2 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 17:00:24 +0000 Subject: [PATCH 28/72] Change the signature of the xUnfetch method to "int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p)". FossilOrigin-Name: 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os.c | 4 ++-- src/os.h | 2 +- src/os_unix.c | 18 ++++++++++++------ src/pager.c | 10 +++++----- src/sqlite.h.in | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 919b150ea9..143bb5bf58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. -D 2013-03-25T16:28:54.437 +C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". +D 2013-03-25T17:00:24.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,12 +157,12 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 -F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca +F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 +F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 +F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 +F src/sqlite.h.in b04790da19184e883da788d07688f16230d2cb6b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bbcaab3e80d0ff776d8567094b137d1483b3377b -R fd25f18074b1cc787a38214a338c9127 +P dce35c01a5fe66d2970075b1e3f0376026485e4c +R 12377e21e3793332cede5cbfb008d2ac U dan -Z ae05d407cd866087491e3302af269873 +Z 1c478466922e4f7b4b376664d80146c7 diff --git a/manifest.uuid b/manifest.uuid index 23ee7e3b08..838ea0b60b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file +115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 08e0d42172..f2d85bba4d 100644 --- a/src/os.c +++ b/src/os.c @@ -144,8 +144,8 @@ int sqlite3OsShmMap( int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ return id->pMethods->xFetch(id, iOff, iAmt, pp); } -int sqlite3OsUnfetch(sqlite3_file *id, void *p){ - return id->pMethods->xUnfetch(id, p); +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); } /* diff --git a/src/os.h b/src/os.h index a7dacfb6ea..82e07ee1cb 100644 --- a/src/os.h +++ b/src/os.h @@ -260,7 +260,7 @@ int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -int sqlite3OsUnfetch(sqlite3_file *, void *); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index a97aba87f1..00c0088f8a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4591,13 +4591,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the second argument is non-NULL, then this function releases a -** reference obtained by an earlier call to unixFetch(). Or, if the second -** argument is NULL, then this function is being called to inform the VFS -** layer that, according to POSIX, any existing mapping may now be invalid -** and should be unmapped. +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. */ -static int unixUnfetch(sqlite3_file *fd, void *p){ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ /* If p==0 (unmap the entire file) then there must be no outstanding @@ -4605,6 +4608,9 @@ static int unixUnfetch(sqlite3_file *fd, void *p){ ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + if( p ){ pFd->nFetchOut--; }else{ diff --git a/src/pager.c b/src/pager.c index 927126c393..06a7eefee1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3091,7 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); - if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3870,7 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; @@ -3903,7 +3903,7 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); - sqlite3OsUnfetch(pPager->fd, pPg->pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* @@ -5095,7 +5095,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ if( pPager->bUseFetch ){ - sqlite3OsUnfetch(pPager->fd, 0); + sqlite3OsUnfetch(pPager->fd, 0, 0); } } } @@ -5245,7 +5245,7 @@ int sqlite3PagerAcquire( if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); }else{ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); } if( pPg ){ assert( rc==SQLITE_OK ); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7bbd3fa000..b68287b0e7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -730,7 +730,7 @@ struct sqlite3_io_methods { int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); - int (*xUnfetch)(sqlite3_file*, void *p); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; From a64d5a15b7d85ee49c5632ee37f6c0bd3242d258 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 18:25:49 +0000 Subject: [PATCH 29/72] Simulate OOM errors in the sqlite3OsFetch() function. Run malloc.test as part of the "mmap" permutation. FossilOrigin-Name: 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os.c | 1 + test/permutations.test | 2 +- test/speed1p.test | 1 + 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 143bb5bf58..e0bcdeed1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". -D 2013-03-25T17:00:24.194 +C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. +D 2013-03-25T18:25:49.003 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 +F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dce35c01a5fe66d2970075b1e3f0376026485e4c -R 12377e21e3793332cede5cbfb008d2ac +P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 +R 81bdcfaea4ca0021abe5572b7c1cd51d U dan -Z 1c478466922e4f7b4b376664d80146c7 +Z 1c0203030ba88df7bcf075001cefd19c diff --git a/manifest.uuid b/manifest.uuid index 838ea0b60b..2ac770c1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file +77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file diff --git a/src/os.c b/src/os.c index f2d85bba4d..2ba857e2aa 100644 --- a/src/os.c +++ b/src/os.c @@ -142,6 +142,7 @@ int sqlite3OsShmMap( } int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); return id->pMethods->xFetch(id, iOff, iAmt, pp); } int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ diff --git a/test/permutations.test b/test/permutations.test index 6e6b5c90f7..9ff3a801c2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,7 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..df5abedde3 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,7 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 893c0ffc290bb59bcd8f505bec45087211579525 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 19:05:07 +0000 Subject: [PATCH 30/72] Add a test that simulates an error in mmap(). FossilOrigin-Name: 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 7 +++++-- src/test_syscall.c | 20 +++++++++++++++++++- test/malloc_common.tcl | 2 +- test/sysfault.test | 31 +++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e0bcdeed1d..215ae805d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. -D 2013-03-25T18:25:49.003 +C Add\sa\stest\sthat\ssimulates\san\serror\sin\smmap(). +D 2013-03-25T19:05:07.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 +F src/os_unix.c 6815e3dd9a1507f2e36ba5a8dcd36b67ac60f3b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c 8acb843ddb9928dea8962e31bb09f421a72ffccb F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae +F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -611,7 +611,7 @@ F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 -F test/malloc_common.tcl 2930895b0962823ec679853e67e58dd6d8198b3c +F test/malloc_common.tcl 9a98856549bfb3fab205edbc1317216edc52e70d F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test 708a028d6d373e5b3842e4bdc8ba80998c9a4da6 @@ -747,7 +747,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a -F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f +F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 -R 81bdcfaea4ca0021abe5572b7c1cd51d +P 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b +R 602dbedb509e1b6f815316de036c2541 U dan -Z 1c0203030ba88df7bcf075001cefd19c +Z 3ee6172e1f5d6ed5486bb4fa69d324f2 diff --git a/manifest.uuid b/manifest.uuid index 2ac770c1e3..50a8aea1fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file +6ec7367d8e98425f00eeb8215ca8964313c1d0b7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 00c0088f8a..8175bd5c9a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -444,6 +444,9 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4547,7 +4550,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ return SQLITE_IOERR_MMAP; } @@ -7186,7 +7189,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/test_syscall.c b/src/test_syscall.c index d484f22db4..0b053fc599 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -81,6 +81,7 @@ /* From test1.c */ extern const char *sqlite3TestErrorName(int); +#include #include #include @@ -106,6 +107,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); +static void *ts_mmap(void *, size_t, int, int, int, off_t); struct TestSyscallArray { @@ -131,6 +133,7 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +155,7 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +381,20 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 5937b958f8..2ac619b1df 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -264,7 +264,7 @@ proc faultsim_test_result_int {args} { set t [list $testrc $testresult] set r $args if { ($testnfail==0 && $t != [lindex $r 0]) || [lsearch $r $t]<0 } { - error "nfail=$testnfail rc=$testrc result=$testresult" + error "nfail=$testnfail rc=$testrc result=$testresult list=$r" } } diff --git a/test/sysfault.test b/test/sysfault.test index 07d525ca9e..bf7df5df5f 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -243,5 +243,36 @@ do_faultsim_test 3 -faults vfsfault-* -prep { faultsim_test_result {0 20000} } +#------------------------------------------------------------------------- +# Test errors in mmap(). +# +proc vfsfault_install {} { + test_syscall reset + test_syscall install {mmap} +} + +faultsim_delete_and_reopen +execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults vfsfault-* -prep { + faultsim_restore_and_reopen + file_control_chunksize_test db main 8192 + execsql { + PRAGMA mmap_size = 1000000; + } +} -body { + test_syscall errno mmap EACCES + + execsql { + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 {1 2}} {1 {disk I/O error}} +} + finish_test From b7e3a326fe84ff10e9b34a9c11c32e441ac0fc5b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 20:30:13 +0000 Subject: [PATCH 31/72] Use mremap() on Linux. FossilOrigin-Name: 431aecc8600c29c203546e48d256510510238887 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 31 +++++++++++++++++++------------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 920c0ece2f..02d611b4ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sexperimental-mmap\sbranch. -D 2013-03-25T19:57:26.572 +C Use\smremap()\son\sLinux. +D 2013-03-25T20:30:13.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 54b82f341b3c81630aa8c73efd2effe037b69270 +F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 5062db672c00c3365d51cd6b39815078f5b6b525 -R 868df99cd28788ba5be1e70b11892bee -U drh -Z 7385c5f656a5c7af6e1d7e1f0ea19ddb +P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 +R 18e19b2d7d709a7b2b83b278bfc05f98 +U dan +Z f03c297d3465d1cb9dc069838022accc diff --git a/manifest.uuid b/manifest.uuid index 0965ca8390..e4f7651d39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a607d63f0b6a3d3785e9385187d3e6b92e14fc70 \ No newline at end of file +431aecc8600c29c203546e48d256510510238887 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9d2e2c130b..33be793423 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4544,21 +4544,28 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - unixUnmapfile(pFd); + void *pNew = 0; - if( nMap>0 ){ - void *pNew; - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; +#if defined(__linux__) && defined(_GNU_SOURCE) + if( pFd->pMapRegion && nMap>0 ){ + pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + }else +#endif + { + unixUnmapfile(pFd); + if( nMap>0 ){ + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); } - - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } + + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MMAP; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } return SQLITE_OK; From d1ab8065c1039db43eb413702cafb3baa500d69a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 20:50:25 +0000 Subject: [PATCH 32/72] Add munmap and mremap to the set of os interfaces that can be overloaded in os_unix.c. FossilOrigin-Name: 8776047bd776bbf266eb9c3b56683badb84ae73e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 20 +++++++++++++++----- test/syscall.test | 2 +- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 02d611b4ed..b0a5aaeb5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smremap()\son\sLinux. -D 2013-03-25T20:30:13.313 +C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. +D 2013-03-25T20:50:25.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 +F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -747,7 +747,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test cf59bafccb143f8df21c043b3201d5afa9fe2474 +F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 -R 18e19b2d7d709a7b2b83b278bfc05f98 -U dan -Z f03c297d3465d1cb9dc069838022accc +P 431aecc8600c29c203546e48d256510510238887 +R 420b4ff666854b5074068c14722ea276 +U drh +Z feb03effee9a1e5215de46a6338d0c52 diff --git a/manifest.uuid b/manifest.uuid index e4f7651d39..5b13e95f75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -431aecc8600c29c203546e48d256510510238887 \ No newline at end of file +8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 33be793423..625f413406 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -447,6 +447,16 @@ static struct unix_syscall { { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if defined(__linux__) && defined(_GNU_SOURCE) + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4005,7 +4015,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4278,7 +4288,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4501,7 +4511,7 @@ static int unixShmUnmap( static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ - munmap(pFd->pMapRegion, pFd->mmapOrigsize); + osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapOrigsize = 0; @@ -4548,7 +4558,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ #if defined(__linux__) && defined(_GNU_SOURCE) if( pFd->pMapRegion && nMap>0 ){ - pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); }else #endif { @@ -7196,7 +7206,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/test/syscall.test b/test/syscall.test index b9a3b196e3..5bf1225867 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs fchown umask mmap + statvfs fchown umask mmap munmap mremap } { if {[test_syscall exists $s]} {lappend syscall_list $s} } From 0d0614bdc6e59c1cb52bc79fdf8dafbbc78f57f9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 23:09:28 +0000 Subject: [PATCH 33/72] Memory-mapped I/O is now on by default. The "PRAGMA mmap_limit(N)" can be used to issue a hint to the VFS to limit mmap space to N bytes. The VFS is free to ignore that hint if desired. However, if "PRAGMA mmap_limit(0)" is used, xFetch is never called. FossilOrigin-Name: 1b37c4effdd03aa2ea938a71b4f22ed27391689b --- manifest | 42 +++++++++++++++++++++--------------------- manifest.uuid | 2 +- src/btree.c | 4 ++-- src/btree.h | 2 +- src/os_unix.c | 15 ++++++++------- src/pager.c | 22 +++++++--------------- src/pager.h | 2 +- src/pragma.c | 25 +++++++++++-------------- src/sqlite.h.in | 8 ++++---- src/sqliteInt.h | 1 - src/sqliteLimit.h | 7 +++++++ test/dbstatus2.test | 2 +- test/incrblob.test | 8 ++++---- test/mmap1.test | 21 ++++++++++----------- test/permutations.test | 6 +++--- test/speed1p.test | 1 - test/sysfault.test | 3 +-- test/wal5.test | 5 ++--- 18 files changed, 84 insertions(+), 92 deletions(-) diff --git a/manifest b/manifest index b0a5aaeb5f..65993928dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. -D 2013-03-25T20:50:25.204 +C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. +D 2013-03-25T23:09:28.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ef238f2d88c6072cc0d6cff33b72c2a9044a15e4 -F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b +F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 +F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 +F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 457eb0ba9fbc331281dca53182c90035b087527f +F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 -F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d +F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 +F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 +F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -658,7 +658,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 +F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -731,7 +731,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -748,7 +748,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 -F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 +F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -955,7 +955,7 @@ F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 +F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 431aecc8600c29c203546e48d256510510238887 -R 420b4ff666854b5074068c14722ea276 +P 8776047bd776bbf266eb9c3b56683badb84ae73e +R 1fba4ef23570acddc66df7d627b1cb5f U drh -Z feb03effee9a1e5215de46a6338d0c52 +Z c62aae7ee656f659e8aebd3681920a9a diff --git a/manifest.uuid b/manifest.uuid index 5b13e95f75..8e0647f651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file +1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 59970ded7f..04cc20d5c5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2134,11 +2134,11 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** Change the limit on the amount of the database file that may be ** memory mapped. */ -int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); sqlite3BtreeLeave(p); return SQLITE_OK; } diff --git a/src/btree.h b/src/btree.h index f2cca2fc71..23f6ad7068 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,7 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); -int sqlite3BtreeSetMmapSize(Btree*, int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os_unix.c b/src/os_unix.c index 625f413406..8a8516bcda 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -225,6 +225,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -251,11 +256,6 @@ struct unixFile { unsigned char inNormalWrite; /* True if in a normal write operation */ #endif - sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ - void *pMapRegion; /* Memory mapped region */ - int nFetchOut; /* Number of outstanding xFetch refs */ #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3699,7 +3699,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_SIZE: { + case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } @@ -4528,7 +4528,7 @@ static void unixUnmapfile(unixFile *pFd){ ** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the ** created mapping is either the requested size or the value configured -** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not ** recreated as a result of outstanding references) or an SQLite error @@ -4969,6 +4969,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index b7534deb09..4cce844237 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,8 +657,8 @@ struct Pager { char dbFileVers[16]; /* Changes whenever database file changes */ u8 bUseFetch; /* True to use xFetch() */ - int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 mxMmap; /* Desired maximum mmap size */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members @@ -3354,23 +3354,15 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. +** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. */ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ - pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; if( pPager->bUseFetch ){ - void *p; - i64 nMapLimit; - if( pPager->nMapCfgLimit<0 ){ - nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; - } - - p = (void *)&nMapLimit; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } } @@ -3378,8 +3370,8 @@ static void pagerFixMaplimit(Pager *pPager){ /* ** Change the maximum size of any memory mapping made of the database file. */ -void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ - pPager->nMapCfgLimit = nMap; +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ + pPager->mxMmap = mxMmap; pagerFixMaplimit(pPager); } diff --git a/src/pager.h b/src/pager.h index 81ab30c115..6f659136e2 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,7 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); -void sqlite3PagerSetMmapsize(Pager *, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/pragma.c b/src/pragma.c index 9b31797d3f..cd1cfbe213 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -745,26 +745,23 @@ void sqlite3Pragma( }else /* - ** PRAGMA [database.]mmap_size - ** PRAGMA [database.]mmap_size=N + ** PRAGMA [database.]mmap_limit(N) ** - ** Used to set or query the mapping size limit. The mapping size limit is + ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. If it is set to a positive value, then it is - ** interpreted as a maximum size in pages. If set to less than zero, then - ** the absolute value is interpreted as a size limit in KB. + ** is not used at all. The parameter N is measured in bytes. ** - ** The default value is zero (do not use memory mapped IO). + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. */ - if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( !zRight ){ - returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); - }else{ - int size = sqlite3Atoi(zRight); - pDb->pSchema->mmap_size = size; - sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + if( zRight ){ + sqlite3_int64 size; + sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); + sqlite3BtreeSetMmapLimit(pDb->pBt, size); } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 72f8d32939..acb49bf5f9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -886,9 +886,9 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** -**
  • [[SQLITE_FCNTL_MMAP_SIZE]] -** The argument is assumed to point to a value of type sqlite3_int64. An -** advisory maximum amount of this file to memory map in bytes. +**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] +** The argument is assumed to pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. ** ** */ @@ -908,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_MMAP_LIMIT 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 59483bc87b..129c4c5ea5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,7 +746,6 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ - int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index c7aee53ceb..2013139456 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,3 +206,10 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif + +/* +** Default maximum size of memory used by xFetch in the VFS. +*/ +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#endif diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 2541a1a823..c2d9164fa2 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,7 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/incrblob.test b/test/incrblob.test index 4277e5c4c1..fa564de0cf 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,7 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -150,7 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -173,7 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -203,7 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/mmap1.test b/test/mmap1.test index 41714e29d8..d97ada7e57 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,16 +23,16 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = -65536 } 4 {} - 1.2 { PRAGMA mmap_size = -50 } 156 {} - 1.3 { PRAGMA mmap_size = 0 } 344 {} - 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} - 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} +foreach {t mmap_limit nRead c2init} { + 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} + 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } + 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } } { do_multiclient_test tn { - sql1 $mmap_size + sql1 $mmap_limit sql2 $c2init code2 { @@ -78,7 +78,7 @@ foreach {t mmap_size nRead c2init} { } {64 ok 149} # Check that the number of pages read by connection 1 indicates that the - # "PRAGMA mmap_size" command worked. + # "PRAGMA mmap_limit" command worked. do_test $t.$tn.5 { nRead db } $nRead } } @@ -95,7 +95,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = -65536; + PRAGMA mmap_limit = 70000000; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -130,4 +130,3 @@ do_execsql_test 2.4 { } {0 24 24} finish_test - diff --git a/test/permutations.test b/test/permutations.test index 9ff3a801c2..2e529fa880 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "mmap" -prefix "mm-" -description { - Similar to veryquick. Except with memory mapping enabled. +test_suite "no-mmap" -prefix "nomm-" -description { + Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = -65536; + pragma mmap_size = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/speed1p.test b/test/speed1p.test index df5abedde3..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,7 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; diff --git a/test/sysfault.test b/test/sysfault.test index bf7df5df5f..4ecaa1fabc 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -262,7 +262,7 @@ do_faultsim_test 4 -faults vfsfault-* -prep { faultsim_restore_and_reopen file_control_chunksize_test db main 8192 execsql { - PRAGMA mmap_size = 1000000; + PRAGMA mmap_limit = 1000000; } } -body { test_syscall errno mmap EACCES @@ -275,4 +275,3 @@ do_faultsim_test 4 -faults vfsfault-* -prep { } finish_test - diff --git a/test/wal5.test b/test/wal5.test index d8bf541a4b..ebd1dbfc31 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -241,8 +241,8 @@ foreach {testprefix do_wal_checkpoint} { # [sql2] commmand above. So normally, the db is 1 page in size here. # However, in mmap() mode, the db is pre-allocated to 2 pages at the # start of the checkpoint, even though page 2 cannot be written. - set nDb 1 - if {[permutation]=="mmap"} {set nDb 2} + set nDb 2 + if {[permutation]=="no-mmap"} {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } @@ -351,4 +351,3 @@ foreach {testprefix do_wal_checkpoint} { finish_test - From 41f89cc6763b71bcad37964745c1ffdf2a391452 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 01:07:50 +0000 Subject: [PATCH 34/72] Previous check-in accidently left mmap turned off by default. This checkin fixes that. Unfortunately, shared.test is now segfaulting. All other veryquick tests appear to work, however. FossilOrigin-Name: a850c7319c20b5757983443df05cf2aa4250053b --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/pager.c | 1 + test/exclusive2.test | 2 +- test/func.test | 2 +- test/pageropt.test | 2 +- test/permutations.test | 4 ++-- test/wal.test | 4 ++-- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 65993928dd..f9fdab7968 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. -D 2013-03-25T23:09:28.646 +C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. +D 2013-03-26T01:07:50.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 +F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b0fc34fdc36897769651975a2b0a606312753643 +F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -654,11 +654,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 +F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e +F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -951,7 +951,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf +F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8776047bd776bbf266eb9c3b56683badb84ae73e -R 1fba4ef23570acddc66df7d627b1cb5f +P 1b37c4effdd03aa2ea938a71b4f22ed27391689b +R 66cf00b9dd2b30a0d14b8b5c617a6bbb U drh -Z c62aae7ee656f659e8aebd3681920a9a +Z c802a777b217fa0e42ce25209bcf7fef diff --git a/manifest.uuid b/manifest.uuid index 8e0647f651..dc7e369204 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file +a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4cce844237..65969bcc3e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,6 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; *ppPager = pPager; return SQLITE_OK; diff --git a/test/exclusive2.test b/test/exclusive2.test index 54203e3d8f..d09097671d 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -28,7 +28,7 @@ ifcapable {!pager_pragmas} { # Tests in this file verify that locking_mode=exclusive causes SQLite to # use cached pages even if the database is changed on disk. This doesn't # work with mmap. -if {[permutation]=="mmap"} { +if {[permutation]!="nommap"} { finish_test return } diff --git a/test/func.test b/test/func.test index 4ab7688461..032becc94f 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,7 +1273,7 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -if {[permutation] != "mmap"} { +if {[permutation] == "nommap"} { do_test func-29.4 { set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] if {$x>100} {set x many} diff --git a/test/pageropt.test b/test/pageropt.test index de39bbaf05..10d3d4917e 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] +} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index 2e529fa880..3aba9d9b38 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "no-mmap" -prefix "nomm-" -description { +test_suite "nommap" -prefix "nomm-" -description { Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = 0; + pragma mmap_limit = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/wal.test b/test/wal.test index ca7edef952..0539192d25 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,8 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db -set nWal 37 -if {[permutation]=="mmap"} {set nWal 39} +set nWal 39 +if {[permutation]=="nommap"} {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; From 0aed84d19f8e1fcf90340e17cff6b774b0379255 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Mar 2013 14:16:20 +0000 Subject: [PATCH 35/72] In btree.c, save the positions of any open cursors before moving any pages around to auto-vacuum the database on commit. FossilOrigin-Name: 30c0a69363931a72d1c34e5be71646932398d172 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f9fdab7968..7b9df5476e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. -D 2013-03-26T01:07:50.891 +C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. +D 2013-03-26T14:16:20.450 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1b37c4effdd03aa2ea938a71b4f22ed27391689b -R 66cf00b9dd2b30a0d14b8b5c617a6bbb -U drh -Z c802a777b217fa0e42ce25209bcf7fef +P a850c7319c20b5757983443df05cf2aa4250053b +R 211ffabed726a7468ad9f8d3eeabe8a8 +U dan +Z 3ec7da7f1db0278991c8541b22037fbf diff --git a/manifest.uuid b/manifest.uuid index dc7e369204..12eaba3359 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file +30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 04cc20d5c5..7d2f2d3840 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3185,7 +3185,9 @@ static int autoVacuumCommit(BtShared *pBt){ nFree = get4byte(&pBt->pPage1->aData[36]); nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ rc = incrVacuumStep(pBt, nFin, iFree, 1); } @@ -3202,7 +3204,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } From c86e51358b3e6cd21d2b324116933b9fb7943956 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 14:36:11 +0000 Subject: [PATCH 36/72] Change the name of the Pager.pFree field to Pager.pMmapFreelist. FossilOrigin-Name: 611bd824c24a60d298f28705de323fa2e813a308 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7b9df5476e..a2e312e3a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. -D 2013-03-26T14:16:20.450 +C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. +D 2013-03-26T14:36:11.815 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf +F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a850c7319c20b5757983443df05cf2aa4250053b -R 211ffabed726a7468ad9f8d3eeabe8a8 -U dan -Z 3ec7da7f1db0278991c8541b22037fbf +P 30c0a69363931a72d1c34e5be71646932398d172 +R 11ebc3388f5e3939639a27582f5015a5 +U drh +Z 2c5cf3d5be983a0c21f765acf3ca05b3 diff --git a/manifest.uuid b/manifest.uuid index 12eaba3359..cd71ef1d3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file +611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 65969bcc3e..550c6c7564 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,7 +659,7 @@ struct Pager { u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 mxMmap; /* Desired maximum mmap size */ - PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3854,9 +3854,9 @@ static int pagerAcquireMapPage( ){ PgHdr *p; /* Memory mapped page to return */ - if( pPager->pFree ){ - *ppPage = p = pPager->pFree; - pPager->pFree = p->pDirty; + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; p->pDirty = 0; memset(p->pExtra, 0, pPager->nExtra); }else{ @@ -3891,20 +3891,20 @@ static int pagerAcquireMapPage( static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; - pPg->pDirty = pPager->pFree; - pPager->pFree = pPg; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* -** Free all PgHdr objects stored in the Pager.pFree list. +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. */ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; - for(p=pPager->pFree; p; p=pNext){ + for(p=pPager->pMmapFreelist; p; p=pNext){ pNext = p->pDirty; sqlite3_free(p); } From 6c5696381e5720eaa9bd6c780bd75f5669f04c96 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 18:48:11 +0000 Subject: [PATCH 37/72] Fix a comment in os_unix.c. No code changes. FossilOrigin-Name: 72813b8ec924b91583c679668f7c4561dff82a02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a2e312e3a6..715e199440 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. -D 2013-03-26T14:36:11.815 +C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. +D 2013-03-26T18:48:11.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 +F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 30c0a69363931a72d1c34e5be71646932398d172 -R 11ebc3388f5e3939639a27582f5015a5 +P 611bd824c24a60d298f28705de323fa2e813a308 +R 8f591838a5bc479ac7493be19f46439f U drh -Z 2c5cf3d5be983a0c21f765acf3ca05b3 +Z be06b34348d93c4a77e7034db0a8802f diff --git a/manifest.uuid b/manifest.uuid index cd71ef1d3b..855425c986 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file +72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8a8516bcda..f91f4c4e7a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3107,8 +3107,8 @@ static int unixRead( ); #endif - /* Deal with as much of this write request as possible by transfering - ** data to the memory mapping using memcpy(). */ + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); From 71be5c5c876747be8a8352170519271a20b7ca9f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:29:33 +0000 Subject: [PATCH 38/72] Fix a problem in btree.c that could cause a crash following an OOM. FossilOrigin-Name: b724aa5889f2e09cf0cbef133a672e6486755104 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 715e199440..8a6c9f5c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. -D 2013-03-26T18:48:11.374 +C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. +D 2013-04-01T14:29:33.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 +F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 611bd824c24a60d298f28705de323fa2e813a308 -R 8f591838a5bc479ac7493be19f46439f -U drh -Z be06b34348d93c4a77e7034db0a8802f +P 72813b8ec924b91583c679668f7c4561dff82a02 +R ff1f51b04ee42accb0ddfde388849db6 +U dan +Z fbb073bc333f3d3b9c496b3226200b28 diff --git a/manifest.uuid b/manifest.uuid index 855425c986..069c9bb2ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file +b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7d2f2d3840..03764a9267 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2596,12 +2596,13 @@ static int btreeSwapOutMmap(BtShared *pBt){ if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + if( rc==SQLITE_OK ){ + if( pCsr->iPage==0 ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[0] = pNew; + releasePage(pPg); } - pCsr->apPage[0] = pNew; - releasePage(pPg); - if( rc!=SQLITE_OK ) return rc; } } } From 0943f0bdc870249bf1d2e8692de9fd2f296190a3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:35:01 +0000 Subject: [PATCH 39/72] Fix a typo in a log message in wal.c. FossilOrigin-Name: 83fc48d16572443373e5de62adbd4cd53c2f7786 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8a6c9f5c8d..2da91083ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. -D 2013-04-01T14:29:33.960 +C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. +D 2013-04-01T14:35:01.078 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 72813b8ec924b91583c679668f7c4561dff82a02 -R ff1f51b04ee42accb0ddfde388849db6 +P b724aa5889f2e09cf0cbef133a672e6486755104 +R 96d4d8288650bd963603cf34d02b8e9c U dan -Z fbb073bc333f3d3b9c496b3226200b28 +Z 48fdc69d5f63c210da250932a430f955 diff --git a/manifest.uuid b/manifest.uuid index 069c9bb2ee..9b3ffd9c61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file +83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4602eb2c34..abbd7032f5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1208,7 +1208,7 @@ finished: */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + pWal->hdr.mxFrame, pWal->zWalName ); } } From f55a4cf80c73937610e8b39d553f2c8aea02af8f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 16:56:41 +0000 Subject: [PATCH 40/72] Ensure that a checkpoint does not use an out-of-date mapping. FossilOrigin-Name: a1040f0397d57855500926494c978623286ddc77 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2da91083ab..334e902f99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. -D 2013-04-01T14:35:01.078 +C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. +D 2013-04-01T16:56:41.264 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf +F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b724aa5889f2e09cf0cbef133a672e6486755104 -R 96d4d8288650bd963603cf34d02b8e9c +P 83fc48d16572443373e5de62adbd4cd53c2f7786 +R 8c156c43b98e65e90e42898654b4787f U dan -Z 48fdc69d5f63c210da250932a430f955 +Z 8dac7ace59503631e98c649abc9ec1b4 diff --git a/manifest.uuid b/manifest.uuid index 9b3ffd9c61..b6b3710dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file +a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index abbd7032f5..881f0a4c78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2956,6 +2956,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ From 5175b324b84ecab7132382bbb8de698988fcd4b9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:22:51 +0000 Subject: [PATCH 41/72] Add xFetch and xUnfetch methods to the os_win.c VFS. FossilOrigin-Name: a1653a257d6af6e8b10c819e68b12f6c2f485811 --- manifest | 14 +-- manifest.uuid | 2 +- src/os_win.c | 288 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 182 insertions(+), 122 deletions(-) diff --git a/manifest b/manifest index 334e902f99..e6eca96ae6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. -D 2013-04-01T16:56:41.264 +C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. +D 2013-04-01T17:22:51.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 +F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83fc48d16572443373e5de62adbd4cd53c2f7786 -R 8c156c43b98e65e90e42898654b4787f -U dan -Z 8dac7ace59503631e98c649abc9ec1b4 +P a1040f0397d57855500926494c978623286ddc77 +R 52b13f77704d4760e77e0cfd9ab57bff +U drh +Z 878a27609130450cef13cc90a8feab48 diff --git a/manifest.uuid b/manifest.uuid index b6b3710dd3..c685b6feac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file +a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index a836664565..b1f5748638 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,9 +150,12 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif - HANDLE hMap; /* Handle for accessing memory mapping */ - void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ }; /* @@ -2066,7 +2069,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ } /* Forward references to VFS methods */ -static int winUnmap(sqlite3_file *); +static int winUnmapfile(winFile*); /* ** Close a file. @@ -2090,7 +2093,7 @@ static int winClose(sqlite3_file *id){ OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); - rc = winUnmap(id); + rc = winUnmapfile(pFile); if( rc!=SQLITE_OK ) return rc; do{ @@ -2142,10 +2145,24 @@ static int winRead( assert( id!=0 ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; @@ -2189,13 +2206,27 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ @@ -2288,14 +2319,14 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, "winTruncate2", pFile->zPath); - }else{ - /* If the file was just truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; - } + } + + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -2805,6 +2836,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_LIMIT: { + pFile->mmapLimit = *(i64*)pArg; + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -3475,37 +3510,25 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ -/* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 -*/ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - /* ** Cleans up the mapped region of the specified file, if any. */ -static int winUnmap( - sqlite3_file *id -){ - winFile *pFile = (winFile*)id; +static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap1", pFile->zPath); } pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap2", pFile->zPath); } pFile->hMap = NULL; @@ -3514,116 +3537,150 @@ static int winUnmap( } /* -** Map, remap or unmap part of the database file. +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. */ -static int winMremap( - sqlite3_file *id, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - winFile *pFile = (winFile*)id; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; HANDLE hMap = NULL; /* New mapping handle */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ - assert( pFile!=0 ); - assert( iOff==0 ); - assert( nOld>=0 ); - assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); - assert( nNew>=0 ); - assert( ppMap ); - assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); - assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); - /* assert( pFile->mmapSize==nOld ); */ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; - assert( winSysInfo.dwPageSize>0 ); - nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); - assert( nNewRnd>=0 ); - nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); - assert( nOldRnd>=0 ); - - if( nNewRnd==nOldRnd ){ - pFile->mmapSize = nNew; - return SQLITE_OK; - } - - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ - sqlite3_int64 oldSz; - rc = winFileSize(id, &oldSz); - if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ - if( rc==SQLITE_OK ){ - rc = winTruncate(id, nNewRnd); - } + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; } - - /* If we get this far, unmap any old mapping. */ - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - - /* And, if required, create a new mapping. */ - if( nNewRnd>0 ){ - i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * - (winSysInfo.dwAllocationGranularity)); + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; + } + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; DWORD protect = PAGE_READONLY; DWORD flags = FILE_MAP_READ; - if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); + hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #endif if( hMap==NULL ){ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap1", pFile->zPath); - return rc; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + return SQLITE_OK; } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); + pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); #else - assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); - pNew = osMapViewOfFile(hMap, flags, - (DWORD)((offset>>32) & 0xffffffff), - (DWORD)(offset & 0xffffffff), - (SIZE_T)nNewRnd); + pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ osCloseHandle(hMap); hMap = NULL; - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap2", pFile->zPath); + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + return SQLITE_OK; } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } - pFile->hMap = hMap; - pFile->pMapRegion = pNew; - pFile->mmapSize = nNew; + return SQLITE_OK; +} - *ppMap = pNew; - return rc; +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -3654,7 +3711,8 @@ static const sqlite3_io_methods winIoMethod = { winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ winShmUnmap, /* xShmUnmap */ - winMremap, /* xMremap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -4052,6 +4110,8 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; + pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; OpenCounter(+1); return rc; From f9d18e472bd853c2849de8aee6b8498f20e254f4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:56:28 +0000 Subject: [PATCH 42/72] Bug fix in the winMapfile() subroutine: Be sure to record the map object handle in the sqlite3_file object. FossilOrigin-Name: ee4d188e207efa24a26776fa4f025c6ac39cce73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..e46bdd10e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Bug\sfix\sin\sthe\swinMapfile()\ssubroutine:\s\sBe\ssure\sto\srecord\sthe\smap\sobject\nhandle\sin\sthe\ssqlite3_file\sobject. +D 2013-04-01T17:56:28.844 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 +F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 88910ffb14df2e25c8fd9ccc5fce98d7 U drh -Z 878a27609130450cef13cc90a8feab48 +Z d2117afc31dea61942683c2e81660a00 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..2f1539dbd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +ee4d188e207efa24a26776fa4f025c6ac39cce73 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index b1f5748638..5ce5a4762a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3555,7 +3555,6 @@ static int winUnmapfile(winFile *pFile){ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ sqlite3_int64 nMap = nByte; int rc; - HANDLE hMap = NULL; /* New mapping handle */ assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -3584,17 +3583,17 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #endif - if( hMap==NULL ){ + if( pFd->hMap==NULL ){ pFd->lastErrno = osGetLastError(); rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); @@ -3603,13 +3602,13 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else - pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ - osCloseHandle(hMap); - hMap = NULL; + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; pFd->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); From e6ecd6630dd7fadec974ffd94ea678212e959717 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 17:56:59 +0000 Subject: [PATCH 43/72] Attempt to emulate mremap() on non-Linux systems by allocating a second mapping immediately following the first in virtual memory. FossilOrigin-Name: 4d67433db8fb4754ae6b192945e479f3d7bad579 --- manifest | 14 +++--- manifest.uuid | 2 +- src/os_unix.c | 137 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 122 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..52c1269c2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Attempt\sto\semulate\smremap()\son\snon-Linux\ssystems\sby\sallocating\sa\ssecond\smapping\simmediately\sfollowing\sthe\sfirst\sin\svirtual\smemory. +D 2013-04-01T17:56:59.408 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff -U drh -Z 878a27609130450cef13cc90a8feab48 +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 144bfd1ec5d6fde06365c0d3ea4b879a +U dan +Z 4e8a79b712821b5d3d9730a626894d79 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..694ae46ba8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +4d67433db8fb4754ae6b192945e479f3d7bad579 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f91f4c4e7a..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -313,6 +313,17 @@ struct unixFile { #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -450,7 +461,7 @@ static struct unix_syscall { { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) -#if defined(__linux__) && defined(_GNU_SOURCE) +#if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, @@ -1124,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -3619,7 +3629,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } - if( pFile->mmapLimit>0 ){ + if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ @@ -4518,6 +4528,104 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif _BSD_SOURCE + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapOrigsize +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapLimit ); + assert( nNew>0 ); + assert( pFd->mmapOrigsize>=pFd->mmapSize ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = MAP_FAILED; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free the existing + ** mapping and set pNew to NULL so that the code below will create a + ** new mapping from scratch. */ + if( pNew==MAP_FAILED ){ + pNew = 0; + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, "mmap", pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } + } + + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapOrigsize = nNew; +} + /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -4554,28 +4662,11 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew = 0; - -#if defined(__linux__) && defined(_GNU_SOURCE) - if( pFd->pMapRegion && nMap>0 ){ - pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); - }else -#endif - { + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ unixUnmapfile(pFd); - if( nMap>0 ){ - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - } } - - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; - } - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } return SQLITE_OK; From db56bcbe42c7d847eda67533717316e4ff6212ae Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:15:50 +0000 Subject: [PATCH 44/72] In windows, round down the size of the memory mapped region to the next smaller multiple of the page size. FossilOrigin-Name: 0e44ed1badde6d766f18d1373d968e5b286b15bd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ed4a17e30f..d43ccc6be3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\saccidental\sfork. -D 2013-04-01T17:58:10.824 +C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. +D 2013-04-01T18:15:50.853 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 +F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4d67433db8fb4754ae6b192945e479f3d7bad579 ee4d188e207efa24a26776fa4f025c6ac39cce73 -R 5abed6f6d2f091dbf577966219204ad2 -U dan -Z 994fbbedca554dbff615409d58ee1a84 +P 5f4437c0e35c8aa740515aac3e81e0146716d2bb +R bde1c152af7414f4a8d87d63f33cdc11 +U drh +Z 9887751c05c44b393c15dab1850774d3 diff --git a/manifest.uuid b/manifest.uuid index 5c2360cd0a..91d9770506 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f4437c0e35c8aa740515aac3e81e0146716d2bb \ No newline at end of file +0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5ce5a4762a..6965ee90bb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3568,6 +3568,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ if( nMap>pFd->mmapLimit ){ nMap = pFd->mmapLimit; } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); if( nMap==0 && pFd->mmapSize>0 ){ winUnmapfile(pFd); From a539c8a4642c91fed157ee48ddf60f92985c5ac9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:25:48 +0000 Subject: [PATCH 45/72] Add a comment to xUnfetch on windows to say that the call to winUnmapfile() might be unnecessary. FossilOrigin-Name: e3c63c31a21b27806592d066a013a111b280c61f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d43ccc6be3..9d43650636 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. -D 2013-04-01T18:15:50.853 +C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. +D 2013-04-01T18:25:48.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 +F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5f4437c0e35c8aa740515aac3e81e0146716d2bb -R bde1c152af7414f4a8d87d63f33cdc11 +P 0e44ed1badde6d766f18d1373d968e5b286b15bd +R 8c3ade0fe410bf8f08819b27491eb958 U drh -Z 9887751c05c44b393c15dab1850774d3 +Z 1d97c5c3c6a55788c16fe769f3e976ae diff --git a/manifest.uuid b/manifest.uuid index 91d9770506..224e9238f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file +e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6965ee90bb..ee7a71018e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3676,6 +3676,10 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ if( p ){ pFd->nFetchOut--; }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ winUnmapfile(pFd); } From a1f42c7c32b7326568305f64cf5c592910928a3e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:38:06 +0000 Subject: [PATCH 46/72] Add the SQLITE_CONFIG_MMAP_LIMIT configuration option for overriding the SQLITE_DEFAULT_MMAP_LIMIT compile-time setting. Enhance "PRAGMA mmap_limit" so that without a specific database name, it sets the limit on all database files and changes the default for any future databases that might be added using ATTACH. FossilOrigin-Name: 78141d0a16dd1d56b575fccd149de7fa789cb06c --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/btree.c | 1 + src/global.c | 1 + src/main.c | 8 ++++++++ src/os_unix.c | 1 + src/pager.c | 2 +- src/pragma.c | 13 +++++++++++-- src/sqlite.h.in | 14 ++++++++++++++ src/sqliteInt.h | 2 ++ 10 files changed, 53 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9d43650636..995cb45fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. -D 2013-04-01T18:25:48.030 +C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. +D 2013-04-01T22:38:06.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 +F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -134,7 +134,7 @@ F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d -F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a +F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -143,7 +143,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 379160ec3680e3009aa4978eac47027c3ef27ac5 +F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec +F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 +F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,10 +176,10 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b +F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0e44ed1badde6d766f18d1373d968e5b286b15bd -R 8c3ade0fe410bf8f08819b27491eb958 +P e3c63c31a21b27806592d066a013a111b280c61f +R 6d1c950d2134f5ef56d487146725ba7a U drh -Z 1d97c5c3c6a55788c16fe769f3e976ae +Z 72c1547cadf3e5127066e34139489015 diff --git a/manifest.uuid b/manifest.uuid index 224e9238f1..838dd7eeb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file +78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 03764a9267..9672e0e000 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1864,6 +1864,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ diff --git a/src/global.c b/src/global.c index f5da7e7f1f..aec3958c1f 100644 --- a/src/global.c +++ b/src/global.c @@ -156,6 +156,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 2fcb236e83..288d4535f3 100644 --- a/src/main.c +++ b/src/main.c @@ -496,6 +496,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_LIMIT: { + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + sqlite3GlobalConfig.mxMmap = mxMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -2316,6 +2323,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->mxMmap = sqlite3GlobalConfig.mxMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..c10841abf9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,6 +3711,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; +printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/pager.c b/src/pager.c index 550c6c7564..68b2e2d667 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,7 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; diff --git a/src/pragma.c b/src/pragma.c index cd1cfbe213..7a515cba2e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -750,7 +750,9 @@ void sqlite3Pragma( ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. The parameter N is measured in bytes. + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** The parameter N is measured in bytes. ** ** This value is advisory. The underlying VFS is free to memory map ** as little or as much as it wants. Except, if N is set to 0 then the @@ -760,8 +762,15 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ sqlite3_int64 size; + int ii; sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - sqlite3BtreeSetMmapLimit(pDb->pBt, size); + if( size<0 ) size = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = size; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + } + } } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index acb49bf5f9..de1316606b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1637,6 +1637,19 @@ struct sqlite3_mem_methods { ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. +** +** [[SQLITE_CONFIG_MMAP_LIMIT]] +**
    SQLITE_CONFIG_MMAP_LIMIT +**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that +** is the default maximum number of bytes of process address space that +** should be used for accessing each database file using memory mapping. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter +** on the [URI filename] when opening the databaes file or by using the +** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the +** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] +** compile-time option. If the argument to this option is negative, then +** the memory map limit is set to the compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1660,6 +1673,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..d894051790 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -834,6 +834,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 mxMmap; /* Default mmap_limit setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2505,6 +2506,7 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ From 3861f546e3f3752cdce48fb75e37da7394aba882 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:42:48 +0000 Subject: [PATCH 47/72] Remove a debugging printf() accidently left in the previous check-in. FossilOrigin-Name: 8198cdd8ac5dcc1c677fffa869ac965186b96abf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 995cb45fa0..b2e4dbad76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. -D 2013-04-01T22:38:06.929 +C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. +D 2013-04-01T22:42:48.981 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e3c63c31a21b27806592d066a013a111b280c61f -R 6d1c950d2134f5ef56d487146725ba7a +P 78141d0a16dd1d56b575fccd149de7fa789cb06c +R fed26fbe7b97d6a5db36e7e764991fa9 U drh -Z 72c1547cadf3e5127066e34139489015 +Z c0440f45f7713ec5c0ea751d4cf10d7e diff --git a/manifest.uuid b/manifest.uuid index 838dd7eeb0..ee803ab26e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file +8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c10841abf9..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,7 +3711,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; -printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG From 98d2038fc2325c968620afe515e3a99a8a08f098 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:15:23 +0000 Subject: [PATCH 48/72] Always send the SQLITE_FCNTL_MMAP_LIMIT pragma to the VFS, even if the limit is zero and even if the VFS does not support xFetch(). FossilOrigin-Name: 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b2e4dbad76..3d3492ec7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. -D 2013-04-01T22:42:48.981 +C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). +D 2013-04-02T00:15:23.044 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 +F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 78141d0a16dd1d56b575fccd149de7fa789cb06c -R fed26fbe7b97d6a5db36e7e764991fa9 +P 8198cdd8ac5dcc1c677fffa869ac965186b96abf +R a603a97de401ee9321fd272228a03ad0 U drh -Z c0440f45f7713ec5c0ea751d4cf10d7e +Z 4afc43008f65fbdcf2b0b7902c3ce20b diff --git a/manifest.uuid b/manifest.uuid index ee803ab26e..4e3c99ee29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file +01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 68b2e2d667..e391652f48 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3360,10 +3360,8 @@ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - if( pPager->bUseFetch ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); - } + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } From 092d993cb79d11675246e04b1e15d2c48a825f2a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:41:18 +0000 Subject: [PATCH 49/72] Remove an unreachable condition from pager.c. FossilOrigin-Name: 3628e86bf131cb205f08a4b299d84007cd6b25c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3d3492ec7c..767726d357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). -D 2013-04-02T00:15:23.044 +C Remove\san\sunreachable\scondition\sfrom\spager.c. +D 2013-04-02T00:41:18.590 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 +F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8198cdd8ac5dcc1c677fffa869ac965186b96abf -R a603a97de401ee9321fd272228a03ad0 +P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd +R 683cfbd9fb419480e60db9265a2009fc U drh -Z 4afc43008f65fbdcf2b0b7902c3ce20b +Z c53b9215e567f2251d489b56e620483a diff --git a/manifest.uuid b/manifest.uuid index 4e3c99ee29..cd77251101 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file +3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e391652f48..75d12cdc1e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2936,15 +2936,14 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ u32 iFrame = 0; - if( pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); - } + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); if( rc==SQLITE_OK ){ rc = readDbPage(pPg, iFrame); } From 6d81449fc6cabb4ee186cf72547eb1e949e011ac Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 01:45:10 +0000 Subject: [PATCH 50/72] Reorder two conditions to make coverage testing easier. FossilOrigin-Name: 793ba4e996426522eeaa86589a9e974fa1fc1522 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 767726d357..37da7c88aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\scondition\sfrom\spager.c. -D 2013-04-02T00:41:18.590 +C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. +D 2013-04-02T01:45:10.945 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe +F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd -R 683cfbd9fb419480e60db9265a2009fc +P 3628e86bf131cb205f08a4b299d84007cd6b25c3 +R 5022c5197e31d50bddc957f6984f11f1 U drh -Z c53b9215e567f2251d489b56e620483a +Z 379e6b5efd0a997c87c1f9a5494b7aa5 diff --git a/manifest.uuid b/manifest.uuid index cd77251101..8133886ec9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file +793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 75d12cdc1e..cb2b39a992 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5128,7 +5128,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } From a3a2b3bdc0cfad4e96a547ff953f240ca9845061 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 10:29:13 +0000 Subject: [PATCH 51/72] Proposed template preprocessor magic for activating mmap only on platforms where we know it works. FossilOrigin-Name: d96272f031102514b02bd839aac9e37618872200 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 21 +++++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 37da7c88aa..edce1feec8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. -D 2013-04-02T01:45:10.945 +C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. +D 2013-04-02T10:29:13.935 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 +F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3628e86bf131cb205f08a4b299d84007cd6b25c3 -R 5022c5197e31d50bddc957f6984f11f1 +P 793ba4e996426522eeaa86589a9e974fa1fc1522 +R 2b5ee6ceec98b53d533d4ea410a1bfc7 U drh -Z 379e6b5efd0a997c87c1f9a5494b7aa5 +Z ead310385cbe70188969099b54288940 diff --git a/manifest.uuid b/manifest.uuid index 8133886ec9..6c51e2ad5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file +d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2013139456..6adf3e8726 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -210,6 +210,23 @@ /* ** Default maximum size of memory used by xFetch in the VFS. */ -#ifndef SQLITE_DEFAULT_MMAP_LIMIT -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#ifdef __APPLE__ +# include +# if defined(TARGET_OS_IPHONE) +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif +#endif +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ + || defined(__sun) \ + || defined(__DragonFly__) \ + || defined(__FreeBSD__) \ + || defined(__NetBSD__) \ + || defined(__OpenBSD__) +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +# else +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif #endif From 4ff7bc45c6c6a7156e433ff50395dfe5e2885c38 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 12:04:09 +0000 Subject: [PATCH 52/72] Add test cases for errors in mmap() or mremap() is os_unix.c. FossilOrigin-Name: 3098a3c1e7305033904a496ef534cb312a876fab --- manifest | 19 +++++------ manifest.uuid | 2 +- src/os_unix.c | 26 ++++++++------- src/test_syscall.c | 15 ++++++++- test/mmap2.test | 79 ++++++++++++++++++++++++++++++++++++++++++++++ test/speed1p.test | 3 +- 6 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 test/mmap2.test diff --git a/manifest b/manifest index edce1feec8..645d467c46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. -D 2013-04-02T10:29:13.935 +C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. +D 2013-04-02T12:04:09.729 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 +F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -631,6 +631,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -731,7 +732,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1040,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 793ba4e996426522eeaa86589a9e974fa1fc1522 -R 2b5ee6ceec98b53d533d4ea410a1bfc7 -U drh -Z ead310385cbe70188969099b54288940 +P d96272f031102514b02bd839aac9e37618872200 +R 323dfbce5d7140c5ce46f22a5112dcc8 +U dan +Z 3d1807f4fa3a8cf06a6141b170b215ce diff --git a/manifest.uuid b/manifest.uuid index 6c51e2ad5a..dfa80de2b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file +3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..07c919c983 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4560,6 +4560,7 @@ static void unixRemapfile( unixFile *pFd, /* File descriptor object */ i64 nNew /* Required mapping size */ ){ + const char *zErr = "mmap"; int h = pFd->h; /* File descriptor open on db file */ u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ @@ -4571,6 +4572,7 @@ static void unixRemapfile( assert( nNew<=pFd->mmapLimit ); assert( nNew>0 ); assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; @@ -4586,12 +4588,13 @@ static void unixRemapfile( #if HAVE_MREMAP pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; #else pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); if( pNew!=MAP_FAILED ){ if( pNew!=pReq ){ osMunmap(pNew, nNew - nReuse); - pNew = MAP_FAILED; + pNew = 0; }else{ pNew = pOrig; } @@ -4602,7 +4605,6 @@ static void unixRemapfile( ** mapping and set pNew to NULL so that the code below will create a ** new mapping from scratch. */ if( pNew==MAP_FAILED ){ - pNew = 0; osMunmap(pOrig, nReuse); } } @@ -4610,18 +4612,18 @@ static void unixRemapfile( /* If pNew is still NULL, try to create an entirely new mapping. */ if( pNew==0 ){ pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); - if( pNew==MAP_FAILED ){ - pNew = 0; - nNew = 0; - unixLogError(SQLITE_OK, "mmap", pFd->zPath); - - /* If the mmap() above failed, assume that all subsequent mmap() calls - ** will probably fail too. Fall back to using xRead/xWrite exclusively - ** in this case. */ - pFd->mmapLimit = 0; - } } + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapOrigsize = nNew; } diff --git a/src/test_syscall.c b/src/test_syscall.c index 0b053fc599..3d117fb2ab 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -108,7 +108,7 @@ static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); static void *ts_mmap(void *, size_t, int, int, int, off_t); - +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -134,6 +134,7 @@ struct TestSyscallArray { /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -156,6 +157,7 @@ struct TestSyscallArray { #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) #define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -395,6 +397,17 @@ static void *ts_mmap( return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); } +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/mmap2.test b/test/mmap2.test new file mode 100644 index 0000000000..d43ea97adf --- /dev/null +++ b/test/mmap2.test @@ -0,0 +1,79 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests the effect of the mmap() or mremap() system calls +# returning an error on the library. +# +# If either mmap() or mremap() fails, SQLite should log an error +# message, then continue accessing the database using read() and +# write() exclusively. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix mmap2 + +if {[test_syscall defaultvfs] != "unix"} { + finish_test + return +} + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + if {[string match os_unix.c* $msg]} { + lappend ::log $msg + } +} + +foreach syscall {mmap mremap} { + test_syscall uninstall + if {[catch {test_syscall install $syscall}]} continue + + for {set i 1} {$i < 20} {incr i} { + reset_db + + test_syscall fault $i 1 + test_syscall errno $syscall ENOMEM + set ::log "" + + do_execsql_test 1.$syscall.$i.1 { + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + } + + set nFail [test_syscall fault 0 0] + + do_execsql_test 1.$syscall.$i.2 { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } {64 ok} + + do_test 1.$syscall.$i.3 { + expr {$nFail==0 || $nFail==1} + } {1} + + do_test 1.$syscall.$i.4.nFail=$nFail { + regexp ".*${syscall}.*" $::log + } [expr $nFail>0] + } +} + +test_syscall uninstall +finish_test + diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..6bf7b10e1f 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -24,6 +24,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl speed_trial_init speed1 +sqlite3_memdebug_vfs_oom_test 0 + # Set a uniform random seed expr srand(0) @@ -78,7 +80,6 @@ do_test speed1p-1.0 { } } {i2a i2b t1 t2} - # 50000 INSERTs on an unindexed table # set list {} From 07fa864d15ae88e9fb999b37128b7698cbea037a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 14:37:40 +0000 Subject: [PATCH 53/72] Fix a faulty assert() in the os_win.c VFS. FossilOrigin-Name: fd6ee54969322203c02ce9bd8744e095faf6a69c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 645d467c46..7730e2979b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. -D 2013-04-02T12:04:09.729 +C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. +D 2013-04-02T14:37:40.848 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 +F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d96272f031102514b02bd839aac9e37618872200 -R 323dfbce5d7140c5ce46f22a5112dcc8 -U dan -Z 3d1807f4fa3a8cf06a6141b170b215ce +P 3098a3c1e7305033904a496ef534cb312a876fab +R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 +U drh +Z 7afe4013f9081817d8523bb6f04fa909 diff --git a/manifest.uuid b/manifest.uuid index dfa80de2b5..c2883df6c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file +fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index ee7a71018e..d66faba4cb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3601,7 +3601,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ /* Log the error, but continue normal operation using xRead/xWrite */ return SQLITE_OK; } - assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); + assert( (nMap % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else From f6653871b218c18667a5eadd6b449c7d5bdb768d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 15:37:30 +0000 Subject: [PATCH 54/72] Add new test file btreefault.test. FossilOrigin-Name: 7fe908afeba16f64edc16824c67b396138581a8f --- manifest | 13 ++++++------ manifest.uuid | 2 +- test/btreefault.test | 50 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 test/btreefault.test diff --git a/manifest b/manifest index 7730e2979b..4e4e7e16f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. -D 2013-04-02T14:37:40.848 +C Add\snew\stest\sfile\sbtreefault.test. +D 2013-04-02T15:37:30.257 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -314,6 +314,7 @@ F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btreefault.test 06899a377f31a8c1a3048ec69831522d4e5c6045 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test e8b18cc61090b6e5e388f54d6b125d711d1b265a @@ -1041,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3098a3c1e7305033904a496ef534cb312a876fab -R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 -U drh -Z 7afe4013f9081817d8523bb6f04fa909 +P fd6ee54969322203c02ce9bd8744e095faf6a69c +R afe8ecd9cb367696b6be602590bc8be6 +U dan +Z 42e6019db12bcc782c7a96e67c5fd3c7 diff --git a/manifest.uuid b/manifest.uuid index c2883df6c5..005403f1ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file +7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file diff --git a/test/btreefault.test b/test/btreefault.test new file mode 100644 index 0000000000..45d326da7f --- /dev/null +++ b/test/btreefault.test @@ -0,0 +1,50 @@ +# 2013 April 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests designed to test the btree.c +# module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix btreefault + +do_test 1-pre1 { + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA journal_mode = DELETE; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + DELETE FROM t1 WHERE rowid%2; + } + faultsim_save_and_close +} {} + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT + sqlite3_step $::STMT +} -body { + execsql { PRAGMA incremental_vacuum = 10 } +} -test { + sqlite3_finalize $::STMT + faultsim_test_result {0 {}} + faultsim_integrity_check +} + +finish_test + From a2c16c084a8a6d6654ac9a8be7bfe5d181638900 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 18:25:04 +0000 Subject: [PATCH 55/72] Fix the build for Mac. FossilOrigin-Name: d36ce0bbad58445c983597ee93438fe07b659cdc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4e4e7e16f5..0d221db3d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\sbtreefault.test. -D 2013-04-02T15:37:30.257 +C Fix\sthe\sbuild\sfor\sMac. +D 2013-04-02T18:25:04.196 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 +F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fd6ee54969322203c02ce9bd8744e095faf6a69c -R afe8ecd9cb367696b6be602590bc8be6 -U dan -Z 42e6019db12bcc782c7a96e67c5fd3c7 +P 7fe908afeba16f64edc16824c67b396138581a8f +R 1e903cf5fbaea8405561162b82cab3c1 +U drh +Z 5ee32a75ebc407142eddeaec11f8a36d diff --git a/manifest.uuid b/manifest.uuid index 005403f1ad..d710366705 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file +d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 6adf3e8726..826d19dd9c 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -211,7 +211,7 @@ ** Default maximum size of memory used by xFetch in the VFS. */ #ifdef __APPLE__ -# include +# include # if defined(TARGET_OS_IPHONE) # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif From d409792838233ae25045e223278a2d195d47d347 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:31:29 +0000 Subject: [PATCH 56/72] Change an assert in pager.c to acknowledge that it is possible for sqlite3PagerRollback() to return SQLITE_CORRUPT. FossilOrigin-Name: d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- test/wal.test | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0d221db3d2..a0457959ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sfor\sMac. -D 2013-04-02T18:25:04.196 +C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. +D 2013-04-02T18:31:29.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 -F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 +F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -953,7 +953,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 +F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7fe908afeba16f64edc16824c67b396138581a8f -R 1e903cf5fbaea8405561162b82cab3c1 -U drh -Z 5ee32a75ebc407142eddeaec11f8a36d +P d36ce0bbad58445c983597ee93438fe07b659cdc +R 338634a1c64bdacb6f3ed638ff14c520 +U dan +Z 2347325502a78b990959c6fe11156215 diff --git a/manifest.uuid b/manifest.uuid index d710366705..b9069347bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file +d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index cb2b39a992..f626a21d89 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6261,7 +6261,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager diff --git a/test/wal.test b/test/wal.test index 0539192d25..f833b6ab1a 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1569,6 +1569,26 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } +reset_db +do_test 25 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + + db close sqlite3_shutdown test_sqlite3_log From db082408c6d68e5c8b6958806c5364bda3849a16 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:33:55 +0000 Subject: [PATCH 57/72] Add test cases to mmap1.test and pagerfault.test. FossilOrigin-Name: 3050136be962427cc4dcced6077ef29b2a941405 --- manifest | 14 +++++------ manifest.uuid | 2 +- test/mmap1.test | 25 +++++++++++++++++++ test/pagerfault.test | 57 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a0457959ef..856014e0b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. -D 2013-04-02T18:31:29.703 +C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. +D 2013-04-02T18:33:55.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -653,7 +653,7 @@ F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 +F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d36ce0bbad58445c983597ee93438fe07b659cdc -R 338634a1c64bdacb6f3ed638ff14c520 +P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 +R 93a80e505d81032f015b51889ab16396 U dan -Z 2347325502a78b990959c6fe11156215 +Z 374a5f4f4b29eb3200a93246d589254e diff --git a/manifest.uuid b/manifest.uuid index b9069347bf..66226dcc40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file +3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index d97ada7e57..e63e09481c 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -129,4 +129,29 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} + +reset_db +db func rblob rblob +do_execsql_test 3.1 { + PRAGMA auto_vacuum = 1; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + + CREATE TABLE t2(a, b, UNIQUE(a, b)); + INSERT INTO t2 SELECT * FROM t1; +} {} + +do_test 3.2 { + set nRow 0 + db eval {SELECT * FROM t2 ORDER BY a, b} { + if {$nRow==4} { db eval { DELETE FROM t1 } } + incr nRow + } + set nRow +} {8} + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index e04e97e4fc..9ea1aa6328 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,5 +1246,62 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } +#------------------------------------------------------------------------- +# +do_test pagerfault-28-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 512; + + PRAGMA journal_mode = wal; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 100000; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + CREATE TABLE t1(a PRIMARY KEY, b); + } + expr {[file size test.db-shm] >= 96*1024} +} {1} +faultsim_save_and_close + +do_faultsim_test pagerfault-28 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA mmap_limit=0 } + + sqlite3 db2 test.db + db2 eval { SELECT count(*) FROM t2 } + + db func a_string a_string + execsql { + BEGIN; + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + } + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT +} -body { + execsql { ROLLBACK } +} -test { + db2 close + sqlite3_finalize $::STMT + catchsql { ROLLBACK } + faultsim_integrity_check +} + finish_test From a9d79ae95d54d4138356ac5801ff98b555bea939 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 2 Apr 2013 20:13:04 +0000 Subject: [PATCH 58/72] Add assert to the Windows VFS pertaining to the size of the SIZE_T type. FossilOrigin-Name: 4eeea58354e21797ca1f4dc4ea785d0330381f0b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 856014e0b4..1ca22b55cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. -D 2013-04-02T18:33:55.243 +C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. +D 2013-04-02T20:13:04.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 +F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 -R 93a80e505d81032f015b51889ab16396 -U dan -Z 374a5f4f4b29eb3200a93246d589254e +P 3050136be962427cc4dcced6077ef29b2a941405 +R 69136e8e9adf1c4824020b7a88c6a9a7 +U mistachkin +Z 0311bae3bb3a8c1122f90eda3126446b diff --git a/manifest.uuid b/manifest.uuid index 66226dcc40..341d71bfa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file +4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d66faba4cb..a978613e2e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3605,6 +3605,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ From 81d1765ad837a99c2ddd9cb2001aba3213779395 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:19:22 +0000 Subject: [PATCH 59/72] Fix a bug in mmap2.test that was leaving an invalid sqlite3_log() callback installed, causing a crash. FossilOrigin-Name: c0cdaa07355f51bd217881b5b91ec2a609fa56f4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap2.test | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1ca22b55cf..aa9f6ee2cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. -D 2013-04-02T20:13:04.021 +C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. +D 2013-04-02T20:19:22.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -632,7 +632,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 -F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 +F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3050136be962427cc4dcced6077ef29b2a941405 -R 69136e8e9adf1c4824020b7a88c6a9a7 -U mistachkin -Z 0311bae3bb3a8c1122f90eda3126446b +P 4eeea58354e21797ca1f4dc4ea785d0330381f0b +R 6d15151e9d49c450ba09b87b5b0a03e1 +U dan +Z 90fc4c0a95395512a6e1551c81a3562f diff --git a/manifest.uuid b/manifest.uuid index 341d71bfa9..6ec6c81697 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file +c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file diff --git a/test/mmap2.test b/test/mmap2.test index d43ea97adf..fb36507efd 100644 --- a/test/mmap2.test +++ b/test/mmap2.test @@ -74,6 +74,10 @@ foreach syscall {mmap mremap} { } } +db close test_syscall uninstall +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test From c3d53189fca4ec61a2f697ecc23bfb16328360b0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 20:29:33 +0000 Subject: [PATCH 60/72] Adjustments to mmap1.test so that it works better on windows. It still gets some answers wrong, but it no longer crashes. FossilOrigin-Name: ee8d3ceeec40d84adb8798f084b6f1215ab56d92 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap1.test | 19 +++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index aa9f6ee2cd..abae49480a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. -D 2013-04-02T20:19:22.780 +C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. +D 2013-04-02T20:29:33.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 +F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4eeea58354e21797ca1f4dc4ea785d0330381f0b -R 6d15151e9d49c450ba09b87b5b0a03e1 -U dan -Z 90fc4c0a95395512a6e1551c81a3562f +P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 +R 2d6cafbf0f715d3130e0ad2821f95af0 +U drh +Z eb8137777309edfff1693e4a3a7e15de diff --git a/manifest.uuid b/manifest.uuid index 6ec6c81697..6a1646888a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file +ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index e63e09481c..f3e7268105 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -20,18 +20,20 @@ proc nRead {db} { db_enter $db array set stats [btree_pager_stats $bt] db_leave $db + # puts [array get stats] return $stats(read) } foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} - 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {} 1.3 { PRAGMA mmap_limit = 0 } 344 {} - 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } - 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } - 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } + 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } + 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } } { do_multiclient_test tn { + sql1 {PRAGMA page_size=1024} sql1 $mmap_limit sql2 $c2init @@ -45,7 +47,8 @@ foreach {t mmap_limit nRead c2init} { db2 func rblob rblob } - sql2 { + sql2 { + PRAGMA page_size=1024; PRAGMA auto_vacuum = 1; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -95,7 +98,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_limit = 70000000; + PRAGMA mmap_limit = 67108864; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -129,7 +132,7 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} - +db2 close reset_db db func rblob rblob do_execsql_test 3.1 { From 48ccef80595731d2c2b6d759455837db3113f647 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:55:01 +0000 Subject: [PATCH 61/72] Fix a resource leak in os_unix.c. FossilOrigin-Name: b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 6 ++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index abae49480a..041087f917 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. -D 2013-04-02T20:29:33.219 +C Fix\sa\sresource\sleak\sin\sos_unix.c. +D 2013-04-02T20:55:01.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 +F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 -R 2d6cafbf0f715d3130e0ad2821f95af0 -U drh -Z eb8137777309edfff1693e4a3a7e15de +P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 +R 40861d78d26ec4eddf163e4d116b8b4f +U dan +Z 36ae50dfb6860cc1a436ea46d4e8b8f3 diff --git a/manifest.uuid b/manifest.uuid index 6a1646888a..cb24cad099 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file +b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 07c919c983..581afdbc91 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4601,10 +4601,8 @@ static void unixRemapfile( } #endif - /* The attempt to extend the existing mapping failed. Free the existing - ** mapping and set pNew to NULL so that the code below will create a - ** new mapping from scratch. */ - if( pNew==MAP_FAILED ){ + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ osMunmap(pOrig, nReuse); } } From 85830a70c2d07e345564211e13e7f5e6182f4135 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 00:42:01 +0000 Subject: [PATCH 62/72] Fix the unix driver to check defined(_BSD_SOURCE) rather than just the plain _BSD_SOURCE macro. This fixes the build for OpenBSD. FossilOrigin-Name: 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 041087f917..f907c58839 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sresource\sleak\sin\sos_unix.c. -D 2013-04-02T20:55:01.697 +C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. +D 2013-04-03T00:42:01.544 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 +F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 -R 40861d78d26ec4eddf163e4d116b8b4f -U dan -Z 36ae50dfb6860cc1a436ea46d4e8b8f3 +P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 +R 12d119a8bfdd7260b7f76958004bb415 +U drh +Z 8724856398770e48d9cea9c9c7fa6257 diff --git a/manifest.uuid b/manifest.uuid index cb24cad099..4593a953a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file +1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 581afdbc91..6ca99fa532 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4534,7 +4534,7 @@ static void unixUnmapfile(unixFile *pFd){ static int unixGetPagesize(void){ #if HAVE_MREMAP return 512; -#elif _BSD_SOURCE +#elif defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); From 7d9f3943ba060edd2389b4b2c27db0d02524f11d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 01:26:54 +0000 Subject: [PATCH 63/72] Add the -mmap option to the command-line shell, for setting the default mmap_limit. FossilOrigin-Name: fc30d06c94c8212abb0477fb4cec4520d05bea34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f907c58839..0b74d42186 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. -D 2013-04-03T00:42:01.544 +C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. +D 2013-04-03T01:26:54.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca +F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 -R 12d119a8bfdd7260b7f76958004bb415 +P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 +R 414201a51e65d2b5d08b22317803af41 U drh -Z 8724856398770e48d9cea9c9c7fa6257 +Z ee0a75b1373c6c61317486123f3b4064 diff --git a/manifest.uuid b/manifest.uuid index 4593a953a2..ac22576f6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file +fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index c6d7fa3d53..6a0e047475 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1552,6 +1552,43 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -3026,6 +3059,8 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ @@ -3111,6 +3146,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE From 7e36962e6c152768483b9282149ff975359d7ed4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 03:53:15 +0000 Subject: [PATCH 64/72] Make use of xFetch, if it is available when doing a VACUUM or a backup. FossilOrigin-Name: 83bc37af07857960c11275891f853a358dcbbf05 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/backup.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0b74d42186..cbde44af45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. -D 2013-04-03T01:26:54.789 +C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. +D 2013-04-03T03:53:15.647 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 -R 414201a51e65d2b5d08b22317803af41 +P fc30d06c94c8212abb0477fb4cec4520d05bea34 +R 8f21f6233365f9239c040421dd942d16 U drh -Z ee0a75b1373c6c61317486123f3b4064 +Z fab31801d3b721c34a1407901bd055e3 diff --git a/manifest.uuid b/manifest.uuid index ac22576f6d..847cb431dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file +83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..252f61cfca 100644 --- a/src/backup.c +++ b/src/backup.c @@ -397,7 +397,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); From 2b8246e3e23cb98b6d7339de019d446aa3bc3bb8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 10:50:02 +0000 Subject: [PATCH 65/72] Initialize the mmap_limit of temporary files to the configured mmap_limit. FossilOrigin-Name: 24bab7596bb7385981a5d331df5eeb05353547f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/os_win.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbde44af45..78b0bffaa7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. -D 2013-04-03T03:53:15.647 +C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. +D 2013-04-03T10:50:02.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,8 +160,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e -F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf +F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 +F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fc30d06c94c8212abb0477fb4cec4520d05bea34 -R 8f21f6233365f9239c040421dd942d16 +P 83bc37af07857960c11275891f853a358dcbbf05 +R 32aa00c0c8e64a488f9b37d8aa36924f U drh -Z fab31801d3b721c34a1407901bd055e3 +Z db8bee74489339af40a69aa3740a7e67 diff --git a/manifest.uuid b/manifest.uuid index 847cb431dc..56b76309a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file +24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6ca99fa532..39a31cb188 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5060,7 +5060,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/os_win.c b/src/os_win.c index a978613e2e..74aede5704 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -4116,7 +4116,7 @@ static int winOpen( pFile->pMapRegion = 0; pFile->mmapSize = 0; pFile->mmapOrigsize = 0; - pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; From 227a1c482cb2e725712f5a5f75fa0fd977a586ef Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:17:39 +0000 Subject: [PATCH 66/72] In btree.c, save the positions of other cursors open on the same table when writing via an incremental blob handle. Otherwise, they may be left holding an out-of-date xFetch page reference. FossilOrigin-Name: 3f09fba18f7e61e21381ffea13c31b968efd7d77 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 9 +++++++++ test/mmap1.test | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 78b0bffaa7..a260417387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. -D 2013-04-03T10:50:02.049 +C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. +D 2013-04-03T11:17:39.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a +F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 +F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83bc37af07857960c11275891f853a358dcbbf05 -R 32aa00c0c8e64a488f9b37d8aa36924f -U drh -Z db8bee74489339af40a69aa3740a7e67 +P 24bab7596bb7385981a5d331df5eeb05353547f7 +R 5d2a80f93297e2bae93eb7d43d59c6c0 +U dan +Z a5b2fa9ebb6e4e86d546b82b43bf4939 diff --git a/manifest.uuid b/manifest.uuid index 56b76309a3..d1325fe2f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file +3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9672e0e000..a1877ebfe8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8400,6 +8400,15 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + */ + rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + if( rc!=SQLITE_OK ){ + return SQLITE_OK; + } + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/test/mmap1.test b/test/mmap1.test index f3e7268105..802b921048 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -157,4 +157,55 @@ do_test 3.2 { set nRow } {8} +#------------------------------------------------------------------------- +# Ensure that existing cursors using xFetch() pages see changes made +# to rows using the incrblob API. +# +reset_db +set aaa [string repeat a 400] +set bbb [string repeat b 400] +set ccc [string repeat c 400] +set ddd [string repeat d 400] +set eee [string repeat e 400] + +do_execsql_test 4.1 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + SELECT * FROM t1; + BEGIN; +} [list $aaa $bbb $ccc $ddd] + +do_test 4.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_test 4.3 { + foreach r {2 3 4} { + set fd [db incrblob t1 x $r] + puts -nonewline $fd $eee + close $fd + } + + set res [list] + while {"SQLITE_ROW" == [sqlite3_step $::STMT]} { + lappend res [sqlite3_column_text $::STMT 0] + } + set res +} [list $eee $eee $eee] + +do_test 4.4 { + sqlite3_finalize $::STMT +} SQLITE_OK + +do_execsql_test 4.5 { COMMIT } + finish_test + + + From f7679ad19eaaf249bd803010eab2da14f21c61e9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:38:36 +0000 Subject: [PATCH 67/72] When moving a page to make way for the root page of a new table or index in an auto-vacuum database, save the positions of any cursors that may be holding xFetch references to the page being moved. FossilOrigin-Name: 9d9b1da54a555e8fb6037d63d1952458c12956d2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 ++++++++ test/mmap1.test | 41 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a260417387..cf7c875169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. -D 2013-04-03T11:17:39.067 +C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. +D 2013-04-03T11:38:36.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d +F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 +F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 24bab7596bb7385981a5d331df5eeb05353547f7 -R 5d2a80f93297e2bae93eb7d43d59c6c0 +P 3f09fba18f7e61e21381ffea13c31b968efd7d77 +R 7ec6dcb52e20c580b46b605bb884042f U dan -Z a5b2fa9ebb6e4e86d546b82b43bf4939 +Z c27841a9d742e32f5c92b396fb592137 diff --git a/manifest.uuid b/manifest.uuid index d1325fe2f5..ab30a2773c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file +9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a1877ebfe8..dd37112c39 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7269,6 +7269,14 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ diff --git a/test/mmap1.test b/test/mmap1.test index 802b921048..d9babe3d86 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -205,7 +205,46 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } +#------------------------------------------------------------------------- +# Ensure that existing cursors holding xFetch() references are not +# confused if those pages are moved to make way for the root page of a +# new table or index. +# +reset_db +do_execsql_test 5.1 { + PRAGMA auto_vacuum = 2; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + + PRAGMA auto_vacuum; + SELECT * FROM t1; +} [list 2 $aaa $bbb $ccc $ddd] + +do_test 5.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_execsql_test 5.3 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('tricked you!'); + INSERT INTO t2 VALUES('tricked you!'); +} + +do_test 5.4 { + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $bbb + +do_test 5.5 { + sqlite3_finalize $::STMT +} SQLITE_OK + finish_test - From 6c96946475fd5b61168ba6b7e43b33e46fee2a9f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:52:16 +0000 Subject: [PATCH 68/72] Remove the restriction on using xFetch to load the root pages of tables and indexes. It appears to have been based on a misconception. FossilOrigin-Name: 5b082efead488a2fccc18171e640e0aa5252d1d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 33 +-------------------------------- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index cf7c875169..5aaf2a1f4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. -D 2013-04-03T11:38:36.388 +C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. +D 2013-04-03T11:52:16.489 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 +F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3f09fba18f7e61e21381ffea13c31b968efd7d77 -R 7ec6dcb52e20c580b46b605bb884042f +P 9d9b1da54a555e8fb6037d63d1952458c12956d2 +R a9b149f10f100efb054372d5cd9076ff U dan -Z c27841a9d742e32f5c92b396fb592137 +Z cc4101c000adda873dbba809b24aeeeb diff --git a/manifest.uuid b/manifest.uuid index ab30a2773c..a21ce49234 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file +5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dd37112c39..42c4ee3cc0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2583,34 +2583,6 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } -/* -** Ensure that any root page references held by open cursors are not -** mmap pages. -*/ -static int btreeSwapOutMmap(BtShared *pBt){ - int rc = SQLITE_OK; /* Return code */ - BtCursor *pCsr; /* Used to iterate through all open cursors */ - - for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ - if( pCsr->iPage>=0 ){ - MemPage *pPg = pCsr->apPage[0]; - if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ - MemPage *pNew = 0; - rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK ){ - if( pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); - } - pCsr->apPage[0] = pNew; - releasePage(pPg); - } - } - } - } - - return rc; -} - /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2717,9 +2689,6 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); - if( rc==SQLITE_OK ){ - rc = btreeSwapOutMmap(pBt); - } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } @@ -4419,7 +4388,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; From 34f7490311f77d0da8c90260d083f19550fd6c61 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:09:18 +0000 Subject: [PATCH 69/72] Change the mmap_limit pragma to report the new limit, or to report the existing limit if called with no arguments. Report the default mmap_limit as part of PRAGMA compile_options. Set the default mmmap_limit to 0 for all systems other than linux, mac, windows, and solaris. FossilOrigin-Name: 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/ctime.c | 3 +++ src/os_unix.c | 4 +++- src/os_win.c | 4 +++- src/pager.c | 5 +++-- src/pragma.c | 14 +++++++++----- src/sqlite.h.in | 4 +++- src/sqliteLimit.h | 10 +++------- test/mmap1.test | 2 +- 10 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 5aaf2a1f4f..a97a1a1a46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. -D 2013-04-03T11:52:16.489 +C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. +D 2013-04-03T13:09:18.208 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 2a5f251fcd7393808df77ccfc817e7058df08c4c +F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 -F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c -F src/pager.c e26184d451207542d56bd09b84339252a194791e +F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_win.c 1d0ccc1880b626ac2b6868284f6338c90687e8d8 +F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 +F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 +F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e +F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d +F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d9b1da54a555e8fb6037d63d1952458c12956d2 -R a9b149f10f100efb054372d5cd9076ff -U dan -Z cc4101c000adda873dbba809b24aeeeb +P 5b082efead488a2fccc18171e640e0aa5252d1d0 +R 779b4d1891b69fa3c237308161c44940 +U drh +Z c380833e5cb3768ffb9f1e28bd39c0ad diff --git a/manifest.uuid b/manifest.uuid index a21ce49234..a842cd26d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file +2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index c42454ca71..ee6d4cbe2e 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,6 +57,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#ifdef SQLITE_DEFAULT_MMAP_LIMIT + "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif diff --git a/src/os_unix.c b/src/os_unix.c index 39a31cb188..75f9236013 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3710,7 +3710,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*)pArg = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/os_win.c b/src/os_win.c index 74aede5704..dc69b47ae2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2837,7 +2837,9 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*) = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } } diff --git a/src/pager.c b/src/pager.c index f626a21d89..ae2f10069f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3358,9 +3358,10 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ + sqlite3_int64 mx; pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); + mx = pPager->mxMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); } } diff --git a/src/pragma.c b/src/pragma.c index 7a515cba2e..3527c0974c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -759,19 +759,23 @@ void sqlite3Pragma( ** upper layers will never invoke the xFetch interfaces to the VFS. */ if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ + sqlite3_int64 mx; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ - sqlite3_int64 size; int ii; - sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - if( size<0 ) size = sqlite3GlobalConfig.mxMmap; - if( pId2->n==0 ) db->mxMmap = size; + sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); + if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = mx; for(ii=db->nDb-1; ii>=0; ii--){ if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); } } } + mx = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_limit", mx); + } }else /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index de1316606b..a4fb1a8a9a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -888,7 +888,9 @@ struct sqlite3_io_methods { ** **
  • [[SQLITE_FCNTL_MMAP_LIMIT]] ** The argument is assumed to pointer to a value of type sqlite3_int64 that -** is an advisory maximum number of bytes in the file to memory map. +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the original value pointed to is negative. ** ** */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 826d19dd9c..2b02cc5662 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -219,13 +219,9 @@ #ifndef SQLITE_DEFAULT_MMAP_LIMIT # if defined(__linux__) \ || defined(_WIN32) \ - || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ - || defined(__sun) \ - || defined(__DragonFly__) \ - || defined(__FreeBSD__) \ - || defined(__NetBSD__) \ - || defined(__OpenBSD__) -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ # else # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif diff --git a/test/mmap1.test b/test/mmap1.test index d9babe3d86..805fb76525 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -108,7 +108,7 @@ do_execsql_test 2.1 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; -} {wal 0 103 103} +} {67108864 wal 0 103 103} do_execsql_test 2.2 { PRAGMA auto_vacuum; From 716d6d2da760f51b7c22e3091bfa2a35fccdd005 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:38:46 +0000 Subject: [PATCH 70/72] Fix a bug in the detection of iOS when setting the default mmap_limit. FossilOrigin-Name: ac1432b3a8e968be4cbc138b4a35f34187c0c36f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a97a1a1a46..d40b31d70b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. -D 2013-04-03T13:09:18.208 +C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. +D 2013-04-03T13:38:46.133 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 +F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5b082efead488a2fccc18171e640e0aa5252d1d0 -R 779b4d1891b69fa3c237308161c44940 +P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 +R e2d5c255f4ba7e0fb00c083dac088003 U drh -Z c380833e5cb3768ffb9f1e28bd39c0ad +Z 519f2edad4b962613f9e09a344e900e8 diff --git a/manifest.uuid b/manifest.uuid index a842cd26d5..7c7984449a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file +ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2b02cc5662..d3a3b4ac95 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -212,7 +212,7 @@ */ #ifdef __APPLE__ # include -# if defined(TARGET_OS_IPHONE) +# if TARGET_OS_IPHONE # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif #endif From 370c9f498dbb223c05197e532668a6ba5595f78e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 20:04:04 +0000 Subject: [PATCH 71/72] Remove an unreachable branch in the direct blob I/O logic of btree.c. FossilOrigin-Name: f97d7274f48e3bb98ed17eae11abb55064134874 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d40b31d70b..e986a3818e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. -D 2013-04-03T13:38:46.133 +C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. +D 2013-04-03T20:04:04.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 +F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 -R e2d5c255f4ba7e0fb00c083dac088003 +P ac1432b3a8e968be4cbc138b4a35f34187c0c36f +R d830cc1d5cf92f6d3df5e32e65cb6d76 U drh -Z 519f2edad4b962613f9e09a344e900e8 +Z d56f9eb968ec676dc0a522c6c207a8e6 diff --git a/manifest.uuid b/manifest.uuid index 7c7984449a..d557c3a92c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file +f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 42c4ee3cc0..c24481bdf7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8380,11 +8380,13 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ /* Save the positions of all other cursors open on this table. This is ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. */ - rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); - if( rc!=SQLITE_OK ){ - return SQLITE_OK; - } + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); /* Check some assumptions: ** (a) the cursor is open for writing, From b0a8ea046a5083b6f0e862e5e5ddcb50c6957334 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 21:23:28 +0000 Subject: [PATCH 72/72] Release resources prior to bailing out of the btreeCreateTable() routine following an OOM error. FossilOrigin-Name: b2a72be9bab77f050bef75477a278a6294d3e854 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e986a3818e..1dc8e386cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. -D 2013-04-03T20:04:04.697 +C Release\sresources\sprior\sto\sbailing\sout\sof\sthe\sbtreeCreateTable()\sroutine\nfollowing\san\sOOM\serror. +D 2013-04-03T21:23:28.794 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 +F src/btree.c 58c4d68415ca7b13a501b682a9c0ad1e287bcacb F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ac1432b3a8e968be4cbc138b4a35f34187c0c36f -R d830cc1d5cf92f6d3df5e32e65cb6d76 +P f97d7274f48e3bb98ed17eae11abb55064134874 +R 133fe42d87ba7d3deff6d35af20e1f64 U drh -Z d56f9eb968ec676dc0a522c6c207a8e6 +Z 8d00d860d2c818ad1f98631f4c1a926a diff --git a/manifest.uuid b/manifest.uuid index d557c3a92c..a6f66ce7d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file +b2a72be9bab77f050bef75477a278a6294d3e854 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c24481bdf7..2fb5becb57 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7242,12 +7242,11 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** case they are holding a reference to an xFetch reference ** corresponding to page pgnoRoot. */ rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); if( rc!=SQLITE_OK ){ return rc; } - releasePage(pPageMove); - /* Move the page currently at pgnoRoot to pgnoMove. */ rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){