Add some very simple test cases (and resulting bug fixes) for release_memory(). (CVS 2826)

FossilOrigin-Name: 154282fca54bf03d310d6931660f99805bb5477f

manifest

@@ -1,5 +1,5 @@
-C Add\sthe\s(untested)\ssqlite3_release_memory()\sfunction.\s(CVS\s2825)
+C Add\ssome\svery\ssimple\stest\scases\s(and\sresulting\sbug\sfixes)\sfor\srelease_memory().\s(CVS\s2826)
-D 2005-12-18T08:51:23
+D 2005-12-19T14:18:11
 F Makefile.in e3c6b3a38d734d41574c04f2fc90d18de2b87102
 F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -59,8 +59,8 @@ F src/os_unix.c 6394d2fa3a8bfbceb227579b44b4b343b5b54a8f
 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e
 F src/os_win.c 9feb97f49b93d451f8ef7c5dd388e05a44647dc6
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c d501a79293f575065e3ab9169e2dea7b945ce155
+F src/pager.c 154825d3b0cd290bf1eeb2cfa6715fd08941fdec
-F src/pager.h 5c14873902723bf735582d55d7042abfbb3810bc
+F src/pager.h e0acb095b3ad0bca48f2ab00c87346665643f64f
 F src/parse.y 142a4b347c82217332e2d3dfa317ff2b7ac32f9c
 F src/pragma.c 8883b4d34796efa315bdd0ec1b03f580ef1575b9
 F src/prepare.c 1417a396efe55e2767f9f97f694d21b8cac2f4d6
@@ -68,10 +68,10 @@ F src/printf.c f47a2f4b5387cd2ebb12e9117a1a5d6bd9a2b812
 F src/random.c ff5e9a8cad790e2a51cd4d2e7737dc8540e09d1d
 F src/select.c 2292b065bc6be61e01aad39a2e1b93e332fb7e57
 F src/shell.c 4872acee1d2a826c73c914961e469e563204b7f9
-F src/sqlite.h.in addab2c24bc84b4fe74cda7bb102abd34983b14f
+F src/sqlite.h.in 5b16b66db128f5aac5e82721b9012bfa88958bd7
-F src/sqliteInt.h 5581e01a1ca1a9a4d70babb96021740bc17eea2e
+F src/sqliteInt.h 4e822139dc9322795a9644a3d3b281d53f6d99ed
 F src/table.c 486dcfce532685b53b5a2b5da8bba0ded6fb2316
-F src/tclsqlite.c b21dd96ffab1625c986513e003a1945a6ae112ae
+F src/tclsqlite.c ce481c0a21a20641cdfe87c9cbbb328cfb3a58b9
 F src/test1.c d6924b182773b2ad3b22e435e4d3bfd5a846da9e
 F src/test2.c 36390cdfc70c08e5ee0b466d0654a117f398bbff
 F src/test3.c 7c97833e33496c2b69f4fe6b9882ac60a481da97
@@ -82,7 +82,7 @@ F src/tokenize.c 7a3a3d3cc734f684a77c4dfd09eb46fcee25394c
 F src/trigger.c 2925ba96d964d9b717e74006bf7e64b8a6b70d97
 F src/update.c ec8e540617b116725b5a55c8d6b4db8bc67fdd7d
 F src/utf.c b7bffac4260177ae7f83c01d025fe0f5ed70ce71
-F src/util.c 31edbc07f9fb44b23d0c33709ecd50b89ecd96a0
+F src/util.c 0ed80dea0cd12dc09d4c576f6a132fd5cf963b51
 F src/vacuum.c fbfdd3967fd34e2f260fafed88dcbf3c10856b94
 F src/vdbe.c 09aaed71f076bfd4286607ee4845100b910a492f
 F src/vdbe.h 8729a4ee16ff9aeab2af9667df3cf300ff978e13
@@ -182,6 +182,7 @@ F test/malloc.test a5ed721cf7d1b12602ede4f98c11b65ab1582cc0
 F test/malloc2.test e6e321db96d6c94cb18bf82ad7215070c41e624e
 F test/malloc3.test 9797d39eca7087a1022bcc0eb0b22215475c9698
 F test/malloc4.test 2e29d155eb4b4808019ef47eeedfcbe9e09e0f05
+F test/malloc5.test 2418a9d667a50175744ba4c147ee1c75de6cfe5c
 F test/manydb.test d81debbf5871242e3b5df1d3bb5e14c50431b6f8
 F test/memdb.test 1860e060be810bf0775bc57408a5b7c4954bcaea
 F test/memleak.test df2b2b96e77f8ba159a332299535b1e5f18e49ac
@@ -327,7 +328,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P 83c8ae5bee3b6bdb556d2e85fa260ba855742601
+P 345addaa03d3bfa3429a59597fbd3addcff62e30
-R 7b8c8cafe73ba9b7fdaabb41edaaf0e4
+R 4eebf0e3d04c46eed73662dfdeebac6b
 U danielk1977
-Z e796870a052bdfafa8d3fb52aa5cdd1a
+Z 06c0c23d27172e085b09b2f497c7c9da

manifest.uuid

@@ -1 +1 @@
-345addaa03d3bfa3429a59597fbd3addcff62e30
+154282fca54bf03d310d6931660f99805bb5477f

src/pager.c

@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.225 2005/12/18 08:51:23 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.226 2005/12/19 14:18:11 danielk1977 Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -2323,7 +2323,7 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
   ** very slow operation, so we work hard to avoid it.  But sometimes
   ** it can't be helped.
   */
-  if( pPg==0 && syncOk ){
+  if( pPg==0 && pPager->pFirst && syncOk ){
     int rc = syncJournal(pPager);
     if( rc!=0 ){
       sqlite3pager_rollback(pPager);
@@ -2393,12 +2393,14 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){
 ** by the current thread may be sqliteFree()ed.
 **
 ** nReq is the number of bytes of memory required. Once this much has
-** been released, the function returns. The return value is the total number 
+** been released, the function returns. A negative value for nReq means
+** free as much memory as possible. The return value is the total number 
 ** of bytes of memory released.
 */
+#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
 int sqlite3pager_release_memory(int nReq){
   SqliteTsd *pTsd = sqlite3Tsd();
-  Pager *pPager;
+  Pager *p;
   int nReleased = 0;
   int i;
 
@@ -2411,7 +2413,7 @@ int sqlite3pager_release_memory(int nReq){
   for(i=0; i==0 || i==1; i++){
 
     /* Loop through all the SQLite pagers opened by the current thread. */
-    for(pPager=pTsd->pPager; pPager && nReleased<nReq; pPager=pPager->pNext){
+    for(p=pTsd->pPager; p && (nReq<0 || nReleased<nReq); p=p->pNext){
       PgHdr *pPg;
       int rc;
 
@@ -2419,7 +2421,7 @@ int sqlite3pager_release_memory(int nReq){
       ** calling fsync() if this is the first iteration of the outermost 
       ** loop).
       */
-      while( SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) && pPg) {
+      while( SQLITE_OK==(rc = pager_recycle(p, i, &pPg)) && pPg) {
 	/* We've found a page to free. At this point the page has been 
         ** removed from the page hash-table, free-list and synced-list 
 	** (pFirstSynced). It is still in the all pages (pAll) list. 
@@ -2429,10 +2431,11 @@ int sqlite3pager_release_memory(int nReq){
         ** probably is though.
         */
         PgHdr *pTmp;
-        if( pPg==pPager->pAll ){
+        assert( pPg );
-           pPager->pAll = pPg->pNextAll;
+        if( pPg==p->pAll ){
+           p->pAll = pPg->pNextAll;
         }else{
-          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll );
+          for( pTmp=p->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll );
           pTmp->pNextAll = pPg->pNextAll;
         }
         nReleased += sqliteAllocSize(pPg);
@@ -2456,6 +2459,7 @@ int sqlite3pager_release_memory(int nReq){
   
   return nReleased;
 }
+#endif
 
 /*
 ** Acquire a page.

src/pager.h

@@ -13,7 +13,7 @@
 ** subsystem.  The page cache subsystem reads and writes a file a page
 ** at a time and provides a journal for rollback.
 **
-** @(#) $Id: pager.h,v 1.47 2005/12/18 08:51:24 danielk1977 Exp $
+** @(#) $Id: pager.h,v 1.48 2005/12/19 14:18:11 danielk1977 Exp $
 */
 
 #ifndef _PAGER_H_
@@ -108,6 +108,7 @@ int sqlite3pager_rename(Pager*, const char *zNewName);
 void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*);
 int sqlite3pager_movepage(Pager*,void*,Pgno);
 int sqlite3pager_reset(Pager*);
+int sqlite3pager_release_memory(int);
 
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
 int sqlite3pager_lockstate(Pager*);

src/sqlite.h.in

@@ -12,7 +12,7 @@
 ** This header file defines the interface that the SQLite library
 ** presents to client programs.
 **
-** @(#) $Id: sqlite.h.in,v 1.149 2005/12/18 08:51:24 danielk1977 Exp $
+** @(#) $Id: sqlite.h.in,v 1.150 2005/12/19 14:18:11 danielk1977 Exp $
 */
 #ifndef _SQLITE3_H_
 #define _SQLITE3_H_
@@ -1336,6 +1336,8 @@ void *sqlite3_update_hook(
 */
 void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
 
+int sqlite3_release_memory(int);
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.440 2005/12/18 08:51:24 danielk1977 Exp $
+** @(#) $Id: sqliteInt.h,v 1.441 2005/12/19 14:18:11 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -272,7 +272,6 @@ typedef struct SqliteTsd SqliteTsd;
 struct SqliteTsd {
   int isInit;                     /* True if structure has been initialised */
   int mallocFailed;               /* True after a malloc() has failed */
-
 #ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
   unsigned int nSoftHeapLimit;    /* (uint)-1 for unlimited */
   unsigned int nAlloc;            /* Number of bytes currently allocated */

src/tclsqlite.c

@@ -11,7 +11,7 @@
 *************************************************************************
 ** A TCL Interface to SQLite
 **
-** $Id: tclsqlite.c,v 1.140 2005/12/16 06:54:03 danielk1977 Exp $
+** $Id: tclsqlite.c,v 1.141 2005/12/19 14:18:11 danielk1977 Exp $
 */
 #ifndef NO_TCL     /* Omit this whole file if TCL is unavailable */
 
@@ -666,9 +666,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     "exists",             "function",          "last_insert_rowid",
     "nullvalue",          "onecolumn",         "profile",
     "progress",           "rekey",             "rollback_hook",
-    "soft_heap_limit",    "timeout",           "total_changes",
+    "release_memory",     "soft_heap_limit",   "timeout",
-    "trace",              "transaction",       "update_hook",       
+    "total_changes",      "trace",             "transaction",
-    "version",            0                    
+    "update_hook",        "version",           0                    
   };
   enum DB_enum {
     DB_AUTHORIZER,        DB_BUSY,             DB_CACHE,
@@ -678,9 +678,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     DB_EXISTS,            DB_FUNCTION,         DB_LAST_INSERT_ROWID,
     DB_NULLVALUE,         DB_ONECOLUMN,        DB_PROFILE,
     DB_PROGRESS,          DB_REKEY,            DB_ROLLBACK_HOOK,
-    DB_SOFT_HEAP_LIMIT,   DB_TIMEOUT,          DB_TOTAL_CHANGES,    
+    DB_RELEASE_MEMORY,    DB_SOFT_HEAP_LIMIT,  DB_TIMEOUT,
-    DB_TRACE,             DB_TRANSACTION,      DB_UPDATE_HOOK,      
+    DB_TOTAL_CHANGES,     DB_TRACE,            DB_TRANSACTION,
-    DB_VERSION
+    DB_UPDATE_HOOK,       DB_VERSION
   };
   /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */
 
@@ -1754,9 +1754,10 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   **     $db soft_heap_limit N
   **
   ** Set the soft-heap-limit for this thread. Note that the limit is 
-  ** per-thread, not per-database.
+  ** per-thread, not per-database. An empty string is returned.
   */
   case DB_SOFT_HEAP_LIMIT: {
+#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
     int n;
     if( objc!=3 ){
       Tcl_WrongNumArgs(interp, 2, objv, "BYTES");
@@ -1767,6 +1768,34 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     }
     sqlite3_soft_heap_limit(n);
     Tcl_ResetResult(interp);
+#endif
+    break;
+  }
+
+  /*
+  **     $db release_memory ?N?
+  **
+  ** Try to release memory currently held (but not really required) by 
+  ** SQLite database connections opened by the current thread. If an
+  ** integer argument is supplied, then SQLite stops trying to free memory
+  ** after N bytes have been freed.
+  **
+  ** The value returned is the number of bytes actually freed.
+  **/
+  case DB_RELEASE_MEMORY: {
+    int nRelease = 0;
+    int N = -1;
+    if( objc!=2 && objc!=3 ){
+      Tcl_WrongNumArgs(interp, 2, objv, "?N?");
+      return TCL_ERROR;
+    }
+#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
+    if( objc==3 && TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){
+      return TCL_ERROR;
+    }
+    nRelease = sqlite3_release_memory(N);
+#endif
+    Tcl_SetObjResult(interp, Tcl_NewIntObj(nRelease));
     break;
   }
   

src/util.c

@@ -14,7 +14,7 @@
 ** This file contains functions for allocating memory, comparing
 ** strings, and stuff like that.
 **
-** $Id: util.c,v 1.155 2005/12/18 08:51:24 danielk1977 Exp $
+** $Id: util.c,v 1.156 2005/12/19 14:18:11 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -35,7 +35,7 @@
 **     sqlite3AllocSize()
 **
 ** The function sqlite3FreeX performs the same task as sqlite3Free and is
-** guaranteed to be a real function.
+** guaranteed to be a real function. The same holds for sqlite3MallocX
 **
 ** The above APIs are implemented in terms of the functions provided at the Os
 ** level (not in this file). The Os level interface is never accessed directly
@@ -48,7 +48,8 @@
 **
 ** Functions sqlite3MallocRaw() and sqlite3Realloc() may invoke 
 ** sqlite3_release_memory() if a call to sqlite3Os.xMalloc() or
-** sqlite3Os.xRealloc() fails. Function sqlite3Malloc() usually invokes
+** sqlite3Os.xRealloc() fails (or if the soft-heap-limit for the thread is
+** exceeded). Function sqlite3Malloc() usually invokes
 ** sqlite3MallocRaw().
 **
 ** MALLOC TEST WRAPPER ARCHITECTURE
@@ -63,11 +64,6 @@
 **     * Audit outstanding memory allocations (i.e check for leaks).
 */
 
-/*
-** TODO!
-*/
-#define sqlite3_release_memory(x) 0
-
 #ifdef SQLITE_MEMDEBUG
 /*--------------------------------------------------------------------------
 ** Begin code for memory allocation system test layer.
@@ -156,6 +152,37 @@ const char *sqlite3_malloc_id = 0;
   TESTALLOC_STACKSIZE                  /* backtrace() stack */          \
 )
 
+
+#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
+/*
+** Set the soft heap-size limit for the current thread.
+*/
+void sqlite3_soft_heap_limit(int n){
+  unsigned int N;
+  if( n<0 ){
+    /* No limit */
+    N = 0xFFFFFFFF;
+  }else{
+    N = n;
+  }
+  sqlite3Tsd()->nSoftHeapLimit = N;
+}
+
+/*
+** Release memory held by SQLite instances created by the current thread.
+*/
+int sqlite3_release_memory(int n){
+  return sqlite3pager_release_memory(n);
+}
+#else
+/* If SQLITE_OMIT_MEMORY_MANAGEMENT is defined, then define a version
+** of sqlite3_release_memory() to be used by other code in this file.
+** This is done for no better reason than to reduce the number of 
+** pre-processor #ifndef statements.
+*/
+#define sqlite3_release_memory(x) 0    /* 0 == no memory freed */
+#endif
+
 /*
 ** For keeping track of the number of mallocs and frees.   This
 ** is used to check for memory leaks.  The iMallocFail and iMallocReset
@@ -452,7 +479,8 @@ void OSMALLOC_FAILED(){
 
 int OSSIZEOF(void *p){
   if( p ){
-    return sqlite3Os.xAllocationSize(p) - TESTALLOC_OVERHEAD;
+    u32 *pOs = (u32 *)getOsPointer(p);
+    return sqlite3Os.xAllocationSize(pOs) - TESTALLOC_OVERHEAD;
   }
   return 0;
 }
@@ -1264,21 +1292,6 @@ void sqlite3MallocClearFailed(){
   sqlite3Tsd()->mallocFailed = 0;
 }
 
-#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT
-/*
-** Set the soft heap-size limit for the current thread.
-*/
-void sqlite3_soft_heap_limit(int n){
-  unsigned int N;
-  if( n<0 ){
-    /* No limit */
-    N = 0xFFFFFFFF;
-  }else{
-    N = n;
-  }
-  sqlite3Tsd()->nSoftHeapLimit = N;
-}
-#endif
 
 #ifndef NDEBUG
 /*

test/malloc5.test

@@ -0,0 +1,128 @@
+# 2005 November 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# $Id: malloc5.test,v 1.1 2005/12/19 14:18:12 danielk1977 Exp $
+
+#---------------------------------------------------------------------------
+# NOTES ON EXPECTED BEHAVIOUR
+#
+#---------------------------------------------------------------------------
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_test malloc5-1.1 {
+  # Simplest possible test. Call [db release_memory] when there is exactly
+  # one unused page in a single pager cache. This test case set's the 
+  # value of the ::pgalloc variable, which is used in subsequent tests.
+  #
+  # Note: Even though executing this statement on an empty database 
+  # modifies 2 pages (the root of sqlite_master and the new root page), 
+  # the sqlite_master root (page 1) is never freed because the btree layer
+  # retains a reference to it for the entire transaction. 
+  execsql {
+    BEGIN;
+    CREATE TABLE abc(a, b, c);
+  }
+  set ::pgalloc [db release_memory]
+  expr $::pgalloc > 0
+} {1}
+do_test malloc5-1.2 {
+  # Test that the transaction started in the above test is still active.
+  # Because the page freed had been written to, freeing it required a
+  # journal sync and exclusive lock on the database file. Test the file
+  # appears to be locked.
+  sqlite3 db2 test.db
+  catchsql {
+    SELECT * FROM abc;
+  } db2
+} {1 {database is locked}}
+do_test malloc5-1.3 {
+  # Again call [db release_memory] when there is exactly one unused page 
+  # in the cache. The same amount of memory is required, but no journal-sync
+  # or exclusive lock should be established.
+  execsql {
+    COMMIT;
+    BEGIN;
+    SELECT * FROM abc;
+  }
+  db release_memory
+} $::pgalloc
+do_test malloc5-1.4 {
+  # Database should not be locked this time.
+  catchsql {
+    SELECT * FROM abc;
+  } db2
+} {0 {}}
+do_test malloc5-1.5 {
+  # Manipulate the cache so that it contains two unused pages. One requires 
+  # a journal-sync to free, the other does not.
+  execsql {
+    SELECT * FROM abc;
+    CREATE TABLE def(d, e, f);
+  }
+  db release_memory 500
+} $::pgalloc
+do_test malloc5-1.6 {
+  # Database should not be locked this time. The above test case only
+  # requested 500 bytes of memory, which can be obtained by freeing the page
+  # that does not require an fsync().
+  catchsql {
+    SELECT * FROM abc;
+  } db2
+} {0 {}}
+do_test malloc5-1.7 {
+  # Release another 500 bytes of memory. This time we require a sync(), 
+  # so the database file will be locked afterwards.
+  db release_memory 500
+} $::pgalloc
+do_test malloc5-1.8 {
+  catchsql {
+    SELECT * FROM abc;
+  } db2
+} {1 {database is locked}}
+do_test malloc5-1.9 {
+  execsql {
+    COMMIT;
+  }
+} {}
+
+
+do_test malloc5-2.1 {
+  # Put some data in tables abc and def. Both tables are still wholly 
+  # contained within their root pages.
+  execsql {
+    INSERT INTO abc VALUES(1, 2, 3);
+    INSERT INTO abc VALUES(4, 5, 6);
+    INSERT INTO def VALUES(7, 8, 9);
+    INSERT INTO def VALUES(10,11,12);
+  }
+} {}
+do_test malloc5-2.2 {
+  # Load the root-page for table def into the cache. Then query table abc. 
+  # Halfway through the query call sqlite3_release_memory(). The goal of this
+  # test is to make sure we don't free pages that are in use (specifically, 
+  # the root of table abc).
+  set nRelease 0
+  execsql { 
+    BEGIN;
+    SELECT * FROM def;
+  }
+  db eval {SELECT * FROM abc} {
+    incr nRelease [db release_memory]
+    lappend data $a $b $c
+  }
+  list $nRelease $data
+} [list $pgalloc [list 1 2 3 4 5 6]]
+
+finish_test
+
+