Remove unnecessary code from malloc.c. Enhance pcache1.c so that is tries

to reuse existing pages, rather than create new pages, when SQLite is under
memory pressure.  "Memory pressure" means that SQLITE_CONFIG_PAGECACHE memory
is nearly exhausted or sqlite3_soft_heap_limit() has been reached.

FossilOrigin-Name: 51049479a8577e03cc353f71f6e13a10c8323d91

manifest

@@ -1,5 +1,8 @@
-C Further\stests\sfor\sCAST\sin\se_expr.test.
+-----BEGIN PGP SIGNED MESSAGE-----
-D 2010-08-27T11:19:55
+Hash: SHA1
+
+C Remove\sunnecessary\scode\sfrom\smalloc.c.\s\sEnhance\spcache1.c\sso\sthat\sis\stries\nto\sreuse\sexisting\spages,\srather\sthan\screate\snew\spages,\swhen\sSQLite\sis\sunder\nmemory\spressure.\s\s"Memory\spressure"\smeans\sthat\sSQLITE_CONFIG_PAGECACHE\smemory\nis\snearly\sexhausted\sor\ssqlite3_soft_heap_limit()\shas\sbeen\sreached.
+D 2010-08-27T12:21:06
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 543f91f24cd7fee774ecc0a61c19704c0c3e78fd
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -117,7 +120,7 @@ F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff
 F src/btree.c 5047fb303cdf6806a42676a6f513c57e15b7d69b
 F src/btree.h b4ba2fdf6b64c7c376bdfffa826af6b786b151d9
 F src/btreeInt.h 5b034ff54800046cc5870605d683ac1f9134bd99
-F src/build.c 0018d49629fc4807100c988dd191dd95e185bb38
+F src/build.c aa5ebbf9622d28b6674120d851cb26017d0823ec
 F src/callback.c da3c38d0ef5d7f04fae371e519bda61aa9cb1704
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 4f3aadad62c6c9f0d4e5a96718516ac4e3c598df
@@ -137,7 +140,7 @@ F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
 F src/loadext.c 6d422ea91cf3d2d00408c5a8f2391cd458da85f8
 F src/main.c 13c874909c9e2eeb75fe3c7bf021d52a5888acb1
-F src/malloc.c 19a468460c7df72de245f10c06bd0625777b7c83
+F src/malloc.c e81193f1b5ac8ff744477253f1a04e0ac579c597
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2
 F src/mem2.c 9e5f72e38573db9598fe60d3fa530d473cc8714e
@@ -162,7 +165,7 @@ F src/pager.h ef8c8f71ab022cc2fff768a1175dd32355be9dcd
 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58
 F src/pcache.c 1e9aa2dbc0845b52e1b51cc39753b6d1e041cb07
 F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050
-F src/pcache1.c 367d88f7924fee4fd3d23161c4c7552df7139a1f
+F src/pcache1.c 7d94fd171b5f49921f1582655911c5c5aad0f628
 F src/pragma.c 8b24ce00a93de345b6c3bd1e1e2cfba9f63d2325
 F src/prepare.c ce4c35a2b1d5fe916e4a46b70d24a6e997d7c4c6
 F src/printf.c 8ae5082dd38a1b5456030c3755ec3a392cd51506
@@ -173,7 +176,7 @@ F src/select.c 8add6cab889fc02e1492eda8dba462ccf11f51dd
 F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056
 F src/sqlite.h.in 76e41cea494cc3b2397f02ad702be0bee0559bb6
 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89
-F src/sqliteInt.h e33b15e8176442bf7484f0e716edfd1ce03b2979
+F src/sqliteInt.h 78ed6bd32777ad06a8b9910d17307aeeae555485
 F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44
 F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -519,7 +522,7 @@ F test/malloc_common.tcl f4a04b7a733eb114a3da16eb39035cde2c851220
 F test/manydb.test b3d3bc4c25657e7f68d157f031eb4db7b3df0d3c
 F test/memdb.test 0825155b2290e900264daaaf0334b6dfe69ea498
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
-F test/memsubsys1.test 8fb47b7e2523f94c100f5885c5697505524de4b9
+F test/memsubsys1.test 5cdd6aa68f17dafe38549f3c2e62bae1afbd96fd
 F test/memsubsys2.test 72a731225997ad5e8df89fdbeae9224616b6aecc
 F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d
 F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0
@@ -847,7 +850,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P ba6119d1e9300de9ce7448cfa0acd579e8e69e41
+P 9616df8c47ababc2d148f0ab6286efa84bd990fb
-R 7d2f90d4d25f22c0315111d0a66262d0
+R 0d182eef4d36dd7c8dd068fdaf1d817b
-U dan
+U drh
-Z 718c5bf8357d1beb7b1280936b29ea9f
+Z 21746e7d1c6ff945c6448c119d837f57
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQFMd623oxKgR168RlERAmqPAJ0ZTg9D4msTmyXdWHI6l76bjutowgCdHnY4
+cuKPRmabeLtMkcKg8J5zzDs=
+=/Jkm
+-----END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-9616df8c47ababc2d148f0ab6286efa84bd990fb
+51049479a8577e03cc353f71f6e13a10c8323d91

src/build.c

@@ -1648,12 +1648,10 @@ void sqlite3CreateView(
   }
   sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr);
   p = pParse->pNewTable;
-  if( p==0 ){
+  if( p==0 || pParse->nErr ){
     sqlite3SelectDelete(db, pSelect);
     return;
   }
-  assert( pParse->nErr==0 ); /* If sqlite3StartTable return non-NULL then
-                             ** there could not have been an error */
   sqlite3TwoPartName(pParse, pName1, pName2, &pName);
   iDb = sqlite3SchemaToIndex(db, p->pSchema);
   if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName)

src/malloc.c

@@ -97,7 +97,12 @@ static SQLITE_WSD struct Mem0Global {
   ** which pages are available.
   */
   u32 *aScratchFree;
-  u32 *aPageFree;
+
+  /*
+  ** True if heap is nearly "full" where "full" is defined by the
+  ** sqlite3_soft_heap_limit() setting.
+  */
+  int nearlyFull;
 } mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 };
 
 #define mem0 GLOBAL(struct Mem0Global, mem0)
@@ -125,25 +130,24 @@ int sqlite3MallocInit(void){
     sqlite3GlobalConfig.pScratch = 0;
     sqlite3GlobalConfig.szScratch = 0;
   }
-  if( sqlite3GlobalConfig.pPage && sqlite3GlobalConfig.szPage>=512
+  if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512
-      && sqlite3GlobalConfig.nPage>=1 ){
+      || sqlite3GlobalConfig.nPage<1 ){
-    int i;
-    int overhead;
-    int sz = ROUNDDOWN8(sqlite3GlobalConfig.szPage);
-    int n = sqlite3GlobalConfig.nPage;
-    overhead = (4*n + sz - 1)/sz;
-    sqlite3GlobalConfig.nPage -= overhead;
-    mem0.aPageFree = (u32*)&((char*)sqlite3GlobalConfig.pPage)
-                  [sqlite3GlobalConfig.szPage*sqlite3GlobalConfig.nPage];
-    for(i=0; i<sqlite3GlobalConfig.nPage; i++){ mem0.aPageFree[i] = i; }
-    mem0.nPageFree = sqlite3GlobalConfig.nPage;
-  }else{
     sqlite3GlobalConfig.pPage = 0;
     sqlite3GlobalConfig.szPage = 0;
+    sqlite3GlobalConfig.nPage = 0;
   }
   return sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData);
 }
 
+/*
+** Return true if the heap is currently under memory pressure - in other
+** words if the amount of heap used is close to the limit set by
+** sqlite3_soft_heap_limit().
+*/
+int sqlite3HeapNearlyFull(void){
+  return mem0.nearlyFull;
+}
+
 /*
 ** Deinitialize the memory allocation subsystem.
 */
@@ -186,10 +190,13 @@ int sqlite3MemoryAlarm(
   void *pArg,
   sqlite3_int64 iThreshold
 ){
+  int nUsed;
   sqlite3_mutex_enter(mem0.mutex);
   mem0.alarmCallback = xCallback;
   mem0.alarmArg = pArg;
   mem0.alarmThreshold = iThreshold;
+  nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
+  mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed);
   sqlite3_mutex_leave(mem0.mutex);
   return SQLITE_OK;
 }
@@ -240,14 +247,19 @@ static int mallocWithAlarm(int n, void **pp){
   if( mem0.alarmCallback!=0 ){
     int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
     if( nUsed+nFull >= mem0.alarmThreshold ){
+      mem0.nearlyFull = 1;
       sqlite3MallocAlarm(nFull);
+    }else{
+      mem0.nearlyFull = 0;
     }
   }
   p = sqlite3GlobalConfig.m.xMalloc(nFull);
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   if( p==0 && mem0.alarmCallback ){
     sqlite3MallocAlarm(nFull);
     p = sqlite3GlobalConfig.m.xMalloc(nFull);
   }
+#endif
   if( p ){
     nFull = sqlite3MallocSize(p);
     sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull);

src/pcache1.c

@@ -89,6 +89,9 @@ static SQLITE_WSD struct PCacheGlobal {
 
   /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */
   int szSlot;                         /* Size of each free slot */
+  int nSlot;                          /* The number of pcache slots */
+  int nFreeSlot;                      /* Number of unused pcache slots */
+  int nReserve;                       /* Try to keep nFreeSlot above this */
   void *pStart, *pEnd;                /* Bounds of pagecache malloc range */
   PgFreeslot *pFree;                  /* Free page blocks */
   int isInit;                         /* True if initialized */
@@ -136,6 +139,8 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
     PgFreeslot *p;
     sz = ROUNDDOWN8(sz);
     pcache1.szSlot = sz;
+    pcache1.nSlot = pcache1.nFreeSlot = n;
+    pcache1.nReserve = n>90 ? 10 : (n/10 + 1);
     pcache1.pStart = pBuf;
     pcache1.pFree = 0;
     while( n-- ){
@@ -162,6 +167,8 @@ static void *pcache1Alloc(int nByte){
     assert( pcache1.isInit );
     p = (PgHdr1 *)pcache1.pFree;
     pcache1.pFree = pcache1.pFree->pNext;
+    pcache1.nFreeSlot--;
+    assert( pcache1.nFreeSlot>=0 );
     sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
   }else{
 
@@ -195,6 +202,8 @@ static void pcache1Free(void *p){
     pSlot = (PgFreeslot*)p;
     pSlot->pNext = pcache1.pFree;
     pcache1.pFree = pSlot;
+    pcache1.nFreeSlot++;
+    assert( pcache1.nFreeSlot<=pcache1.nSlot );
   }else{
     int iSize;
     assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) );
@@ -280,6 +289,32 @@ void sqlite3PageFree(void *p){
   pcache1LeaveMutex();
 }
 
+
+/*
+** Return true if it desirable to avoid allocating a new page cache
+** entry.
+**
+** If memory was allocated specifically to the page cache using
+** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then
+** it is desirable to avoid allocating a new page cache entry because
+** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient
+** for all page cache needs and we should not need to spill the
+** allocation onto the heap.
+**
+** Or, the heap is used for all page cache memory put the heap is
+** under memory pressure, then again it is desirable to avoid
+** allocating a new page cache entry in order to avoid stressing
+** the heap even further.
+*/
+static int pcache1UnderMemoryPressure(PCache1 *pCache){
+  assert( sqlite3_mutex_held(pcache1.mutex) );
+  if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){
+    return pcache1.nFreeSlot<pcache1.nReserve;
+  }else{
+    return sqlite3HeapNearlyFull();
+  }
+}
+
 /******************************************************************************/
 /******** General Implementation Functions ************************************/
 
@@ -521,14 +556,16 @@ static int pcache1Pagecount(sqlite3_pcache *p){
 **   2. If createFlag==0 and the page is not already in the cache, NULL is
 **      returned.
 **
-**   3. If createFlag is 1, and the page is not already in the cache,
+**   3. If createFlag is 1, and the page is not already in the cache, then
-**      and if either of the following are true, return NULL:
+**      return NULL (do not allocate a new page) if any of the following
+**      conditions are true:
 **
 **       (a) the number of pages pinned by the cache is greater than
 **           PCache1.nMax, or
+**
 **       (b) the number of pages pinned by the cache is greater than
 **           the sum of nMax for all purgeable caches, less the sum of 
-**           nMin for all other purgeable caches. 
+**           nMin for all other purgeable caches, or
 **
 **   4. If none of the first three conditions apply and the cache is marked
 **      as purgeable, and if one of the following is true:
@@ -540,6 +577,9 @@ static int pcache1Pagecount(sqlite3_pcache *p){
 **           already equal to or greater than the sum of nMax for all
 **           purgeable caches,
 **
+**       (c) The system is under memory pressure and wants to avoid
+**           unnecessary pages cache entry allocations
+**
 **      then attempt to recycle a page from the LRU list. If it is the right
 **      size, return the recycled buffer. Otherwise, free the buffer and
 **      proceed to step 5. 
@@ -571,6 +611,7 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
   if( createFlag==1 && (
         nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage)
      || nPinned>=(pCache->nMax * 9 / 10)
+     || pcache1UnderMemoryPressure(pCache)
   )){
     goto fetch_out;
   }
@@ -581,7 +622,9 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){
 
   /* Step 4. Try to recycle a page buffer if appropriate. */
   if( pCache->bPurgeable && pcache1.pLruTail && (
-     (pCache->nPage+1>=pCache->nMax) || pcache1.nCurrentPage>=pcache1.nMaxPage
+         (pCache->nPage+1>=pCache->nMax)
+      || pcache1.nCurrentPage>=pcache1.nMaxPage
+      || pcache1UnderMemoryPressure(pCache)
   )){
     pPage = pcache1.pLruTail;
     pcache1RemoveFromHash(pPage);

src/sqliteInt.h

@@ -2503,6 +2503,7 @@ void sqlite3PageFree(void*);
 void sqlite3MemSetDefault(void);
 void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
 int sqlite3MemoryAlarm(void (*)(void*, sqlite3_int64, int), void*, sqlite3_int64);
+int sqlite3HeapNearlyFull(void);
 
 /*
 ** On systems with ample stack space and that support alloca(), make

test/memsubsys1.test

@@ -99,15 +99,10 @@ build_test_db memsubsys1-2 {PRAGMA page_size=1024}
 #show_memstats
 do_test memsubsys1-2.3 {
   set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
+} 0
-  expr {
-    ($pg_used*1024 + $pg_ovfl) < $max_pagecache &&
-    ($pg_used*(1024+$xtra_size) + $pg_ovfl) >= $max_pagecache
-  }
-} 1
 do_test memsubsys1-2.4 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} 19
+} 20
 do_test memsubsys1-2.5 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
 } 0
@@ -143,7 +138,7 @@ do_test memsubsys1-3.2.3 {
 } 2048
 do_test memsubsys1-3.2.4 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} 19
+} 20
 do_test memsubsys1-3.2.5 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
 } 0
@@ -160,15 +155,11 @@ build_test_db memsubsys1-4 {PRAGMA page_size=1024}
 #show_memstats
 do_test memsubsys1-4.3 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} 49
+  expr {$pg_used>=45 && $pg_used<=50}
+} 1
 do_test memsubsys1-4.4 {
   set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
-  set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
+} 0
-  expr {
-    ($pg_used*1024 + $pg_ovfl) < $max_pagecache &&
-    ($pg_used*(1024+$xtra_size) + $pg_ovfl) >= $max_pagecache
-  }
-} 1
 do_test memsubsys1-4.5 {
   set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
   expr {$maxreq<7000}
@@ -190,7 +181,7 @@ build_test_db memsubsys1-5 {PRAGMA page_size=4096}
 #show_memstats
 do_test memsubsys1-5.3 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} 23
+} 24
 do_test memsubsys1-5.4 {
   set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
   expr {$maxreq>4096}
@@ -216,11 +207,11 @@ build_test_db memsubsys1-6 {PRAGMA page_size=4096}
 #show_memstats
 do_test memsubsys1-6.3 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
-} 23
+} 24
-do_test memsubsys1-6.4 {
+#do_test memsubsys1-6.4 {
-  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
+#  set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-  expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)}
+#  expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)}
-} 1
+#} 1
 do_test memsubsys1-6.5 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
 } 1