Add external locking to test_async.c. There are still some tests to come. (CVS 4398)

FossilOrigin-Name: 3794dcd31a74e90b181b336bf6a4c917bda526b8

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swhereby\sthe\s*ppVtab\soutput\sbuffer\spassed\sto\ssqlite3_module.xConstruct()\scould\sbe\sinvalidated\s(freed)\sif\sa\smalloc()\sfailure\soccured\swithin\sa\scall\sto\ssqlite3_declare_vtab().\s(CVS\s4397)
+C Add\sexternal\slocking\sto\stest_async.c.\sThere\sare\sstill\ssome\stests\sto\scome.\s(CVS\s4398)
-D 2007-09-04T15:38:58
+D 2007-09-04T18:28:44
 F Makefile.in cbfb898945536a8f9ea8b897e1586dd1fdbcc5db
 F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -145,7 +145,7 @@ F src/test6.c 0513982dfef4da2a4154b538d2bf538b84ca21d3
 F src/test7.c a9d509d0e9ad214b4772696f49f6e61be26213d1
 F src/test8.c f113aa3723a52113d0fa7c28155ecd37e7e04077
 F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f
-F src/test_async.c c6f5f75f8ccf5d8be1076c45a9c3d5fd995249b9
+F src/test_async.c e221db3e87b472733a8015be7d70bae0edb848b1
 F src/test_autoext.c 855157d97aa28cf84233847548bfacda21807436
 F src/test_btree.c c1308ba0b88ab577fa56c9e493a09829dfcded9c
 F src/test_config.c 6fb459214b27952b143f45e35200d94096d54cc6
@@ -180,7 +180,7 @@ F test/alter2.test 50c3f554b8236d179d72511c0a4f23c5eb7f2af3
 F test/alter3.test a6eec8f454be9b6ce73d8d7dc711453675a10ce7
 F test/altermalloc.test 1f4d2d66750bea1a78cd9f0b7dba5bfb155dd6cf
 F test/analyze.test 2f55535aa335785db1a2f97d3f3831c16c09f8b0
-F test/async.test b5547f56b329fa2ea5bce80f25c1d91eff36b47b
+F test/async.test 18e7dc66535f3d86c05e0f954384472e2ed52490
 F test/async2.test a8ef7abfda880b171b2f0a8476300816e33a808a
 F test/attach.test b849e1baae863c3a6132ff8b9b1baf356ab6c178
 F test/attach2.test 78bc1a25ea8785c7571b44f5947ada2bd5d78127
@@ -569,7 +569,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 17ca684c124445f17d1e36c37e169056c5fd4569
+P efd61df1b9170f0134787ae17ac996a7eff64add
-R aab5e4f24ec35c5749ccbfc9ee626576
+R ef7fa5443cd985d720e63f8ac953cec0
 U danielk1977
-Z d2e7e7e0b743acfcb2048b0ee18425ea
+Z 605a822a91549091d299d216a22bbd33

manifest.uuid

@@ -1 +1 @@
-efd61df1b9170f0134787ae17ac996a7eff64add
+3794dcd31a74e90b181b336bf6a4c917bda526b8

src/test_async.c

@@ -70,6 +70,13 @@
 ** when the buffer gets to be too big.
 */
 
+/* 
+** If this symbol is defined, then file-system locks are obtained as
+** required. This slows things down, but allows multiple processes
+** to access the database concurrently.
+*/
+#define ENABLE_FILE_LOCKING
+
 #include "sqliteInt.h"
 #include <tcl.h>
 
@@ -79,6 +86,7 @@
 */
 #if OS_UNIX && SQLITE_THREADSAFE
 
+
 /*
 ** This demo uses pthreads.  If you do not have a pthreads implementation
 ** for your operating system, you will need to recode the threading 
@@ -95,6 +103,8 @@
 typedef struct AsyncWrite AsyncWrite;
 typedef struct AsyncFile AsyncFile;
 typedef struct AsyncFileData AsyncFileData;
+typedef struct AsyncFileLock AsyncFileLock;
+typedef struct AsyncLock AsyncLock;
 
 /* Enable for debugging */
 static int sqlite3async_trace = 0;
@@ -132,8 +142,7 @@ static void asyncTrace(const char *zFormat, ...){
 **
 ** File handle operations (invoked by SQLite thread):
 **
-**         asyncWrite, asyncClose, asyncTruncate, asyncSync, 
+**         asyncWrite, asyncClose, asyncTruncate, asyncSync 
-**         asyncSetFullSync, asyncOpenDirectory.
 **    
 **     The operations above add an entry to the global write-op list. They
 **     prepare the entry, acquire the async.queueMutex momentarily while
@@ -159,23 +168,6 @@ static void asyncTrace(const char *zFormat, ...){
 **     and will therefore not honor them.
 **
 **
-**         asyncFileHandle.
-**    
-**     The sqlite3OsFileHandle() function is currently only used when 
-**     debugging the pager module. Unless sqlite3OsClose() is called on the
-**     file (shouldn't be possible for other reasons), the underlying 
-**     implementations are safe to call without grabbing any mutex. So we just
-**     go ahead and call it no matter what any other threads are doing.
-**
-**    
-**         asyncSeek.
-**
-**     Calling this method just manipulates the AsyncFile.iOffset variable. 
-**     Since this variable is never accessed by writer thread, this
-**     function does not require the mutex.  Actual calls to OsSeek() take 
-**     place just before OsWrite() or OsRead(), which are always protected by 
-**     the mutex.
-**
 ** The writer thread:
 **
 **     The async.writerMutex is used to make sure only there is only
@@ -222,7 +214,9 @@ static void asyncTrace(const char *zFormat, ...){
 
 /*
 ** State information is held in the static variable "async" defined
-** as follows:
+** as the following structure.
+**
+** Both async.ioError and async.nFile are protected by async.queueMutex.
 */
 static struct TestAsyncStaticData {
   pthread_mutex_t queueMutex;  /* Mutex for access to write operation queue */
@@ -254,12 +248,13 @@ static struct TestAsyncStaticData {
 #define ASYNC_CLOSE         4
 #define ASYNC_DELETE        5
 #define ASYNC_OPENEXCLUSIVE 6
+#define ASYNC_UNLOCK        7
 
 /* Names of opcodes.  Used for debugging only.
 ** Make sure these stay in sync with the macros above!
 */
 static const char *azOpcodeName[] = {
-  "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX"
+  "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX", "UNLOCK"
 };
 
 /*
@@ -295,6 +290,9 @@ static const char *azOpcodeName[] = {
 **     iOffset -> Value of "delflag".
 **     nByte   -> Number of bytes of zBuf points to (file name).
 **
+** ASYNC_UNLOCK:
+**     nByte   -> Argument to sqlite3OsUnlock().
+**
 **
 ** For an ASYNC_WRITE operation, zBuf points to the data to write to the file. 
 ** This space is sqlite3_malloc()d along with the AsyncWrite structure in a
@@ -316,15 +314,22 @@ struct AsyncWrite {
 ** file was opened with the SQLITE_OPEN_MAIN_DB.
 **
 ** The global async.aLock[] hash table maps from database file-name to a
-** linked-list of AsyncLock structures corresponding to handles opened on the
+** linked-list of AsyncFileLock structures corresponding to handles opened on
-** file. The AsyncLock structures are linked into the list when the file is
+** the file. The AsyncFileLock structures are linked into the list when the
-** opened and removed when it is closed. Mutex async.lockMutex must be held
+** file is opened and removed when it is closed. Mutex async.lockMutex must be
-** before accessing any AsyncLock structure or the async.aLock[] table.
+** held before accessing any AsyncFileLock structure or the async.aLock[]
+** table.
 */
-typedef struct AsyncLock AsyncLock;
+struct AsyncFileLock {
+  int eLock;                /* Internally visible lock state (sqlite pov) */
+  int eAsyncLock;           /* Lock-state with write-queue unlock */
+  AsyncFileLock *pNext;
+};
+
 struct AsyncLock {
+  sqlite3_file *pFile;
   int eLock;
-  AsyncLock *pNext;
+  AsyncFileLock *pList;
 };
 
 /* 
@@ -347,7 +352,7 @@ struct AsyncFileData {
   int nName;                 /* Number of characters in zName */
   sqlite3_file *pBaseRead;   /* Read handle to the underlying Os file */
   sqlite3_file *pBaseWrite;  /* Write handle to the underlying Os file */
-  AsyncLock lock;
+  AsyncFileLock lock;
 };
 
 /*
@@ -478,16 +483,17 @@ static int asyncRead(sqlite3_file *pFile, void *zOut, int iAmt, i64 iOffset){
   int nRead;
   sqlite3_file *pBase = p->pBaseRead;
 
+  /* Grab the write queue mutex for the duration of the call */
+  pthread_mutex_lock(&async.queueMutex);
+
   /* If an I/O error has previously occurred in this virtual file 
   ** system, then all subsequent operations fail.
   */
   if( async.ioError!=SQLITE_OK ){
-    return async.ioError;
+    rc = async.ioError;
+    goto asyncread_out;
   }
 
-  /* Grab the write queue mutex for the duration of the call */
-  pthread_mutex_lock(&async.queueMutex);
-
   if( pBase->pMethods ){
     rc = sqlite3OsFileSize(pBase, &filesize);
     if( rc!=SQLITE_OK ){
@@ -592,6 +598,34 @@ int asyncFileSize(sqlite3_file *pFile, i64 *piSize){
   return rc;
 }
 
+/*
+** Lock or unlock the actual file-system entry.
+*/
+static int getFileLock(AsyncLock *pLock){
+  int rc = SQLITE_OK;
+  AsyncFileLock *pIter;
+  int eRequired = 0;
+
+  if( pLock->pFile ){
+    for(pIter=pLock->pList; pIter; pIter=pIter->pNext){
+      assert(pIter->eAsyncLock>=pIter->eLock);
+      if( pIter->eAsyncLock>eRequired ){
+        eRequired = pIter->eAsyncLock;
+      }
+    }
+    if( eRequired>pLock->eLock ){
+      rc = sqlite3OsLock(pLock->pFile, eRequired);
+    }else if(eRequired<pLock->eLock){
+      rc = sqlite3OsUnlock(pLock->pFile, eRequired);
+    }
+    if( rc==SQLITE_OK ){
+      pLock->eLock = eRequired;
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** No disk locking is performed.  We keep track of locks locally in
 ** the async.aLock hash table.  Locking should appear to work the same
@@ -607,20 +641,28 @@ static int asyncLock(sqlite3_file *pFile, int eLock){
   pthread_mutex_lock(&async.lockMutex);
   if( p->lock.eLock<eLock ){
     AsyncLock *pLock;
+    AsyncFileLock *pIter;
     pLock = (AsyncLock *)sqlite3HashFind(&async.aLock, p->zName, p->nName);
-    assert(pLock);
+    assert(pLock && pLock->pList);
-    for(/*no-op*/; pLock; pLock=pLock->pNext){
+    for(pIter=pLock->pList; pIter; pIter=pIter->pNext){
-      if( pLock!=&p->lock && (
+      if( pIter!=&p->lock && (
-        (eLock==SQLITE_LOCK_EXCLUSIVE && pLock->eLock>=SQLITE_LOCK_SHARED) ||
+        (eLock==SQLITE_LOCK_EXCLUSIVE && pIter->eLock>=SQLITE_LOCK_SHARED) ||
-        (eLock==SQLITE_LOCK_PENDING && pLock->eLock>=SQLITE_LOCK_RESERVED) ||
+        (eLock==SQLITE_LOCK_PENDING && pIter->eLock>=SQLITE_LOCK_RESERVED) ||
-        (eLock==SQLITE_LOCK_RESERVED && pLock->eLock>=SQLITE_LOCK_RESERVED) ||
+        (eLock==SQLITE_LOCK_RESERVED && pIter->eLock>=SQLITE_LOCK_RESERVED) ||
-        (eLock==SQLITE_LOCK_SHARED && pLock->eLock>=SQLITE_LOCK_PENDING)
+        (eLock==SQLITE_LOCK_SHARED && pIter->eLock>=SQLITE_LOCK_PENDING)
       )){
         rc = SQLITE_BUSY;
       }
     }
     if( rc==SQLITE_OK ){
       p->lock.eLock = eLock;
+      if( eLock>p->lock.eAsyncLock ){
+        p->lock.eAsyncLock = eLock;
+      }
+    }
+    assert(p->lock.eAsyncLock>=p->lock.eLock);
+    if( rc==SQLITE_OK ){
+      rc = getFileLock(pLock);
     }
   }
   pthread_mutex_unlock(&async.lockMutex);
@@ -630,13 +672,13 @@ static int asyncLock(sqlite3_file *pFile, int eLock){
 }
 static int asyncUnlock(sqlite3_file *pFile, int eLock){
   AsyncFileData *p = ((AsyncFile *)pFile)->pData;
-  AsyncLock *pLock = &p->lock;
+  AsyncFileLock *pLock = &p->lock;
   pthread_mutex_lock(&async.lockMutex);
   if( pLock->eLock>eLock ){
     pLock->eLock = eLock;
   }
   pthread_mutex_unlock(&async.lockMutex);
-  return SQLITE_OK;
+  return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0);
 }
 
 /*
@@ -645,13 +687,14 @@ static int asyncUnlock(sqlite3_file *pFile, int eLock){
 */
 static int asyncCheckReservedLock(sqlite3_file *pFile){
   int ret = 0;
+  AsyncFileLock *pIter;
   AsyncLock *pLock;
   AsyncFileData *p = ((AsyncFile *)pFile)->pData;
 
   pthread_mutex_lock(&async.lockMutex);
   pLock = (AsyncLock *)sqlite3HashFind(&async.aLock, p->zName, p->nName);
-  for(/*no-op*/; pLock; pLock=pLock->pNext){
+  for(pIter=pLock->pList; pIter; pIter=pIter->pNext){
-    if( pLock->eLock>=SQLITE_LOCK_RESERVED ){
+    if( pIter->eLock>=SQLITE_LOCK_RESERVED ){
       ret = 1;
     }
   }
@@ -665,6 +708,14 @@ static int asyncCheckReservedLock(sqlite3_file *pFile){
 ** This is a no-op, as the asynchronous backend does not support locking.
 */
 static int asyncFileControl(sqlite3_file *id, int op, void *pArg){
+  switch( op ){
+    case SQLITE_FCNTL_LOCKSTATE: {
+      pthread_mutex_lock(&async.lockMutex);
+      *(int*)pArg = ((AsyncFile*)id)->pData->lock.eLock;
+      pthread_mutex_unlock(&async.lockMutex);
+      return SQLITE_OK;
+    }
+  }
   return SQLITE_ERROR;
 }
 
@@ -713,6 +764,8 @@ static int asyncOpen(
   int nByte;
   AsyncFileData *pData;
 
+  AsyncLock *pLock = 0;
+
   nByte = (
     sizeof(AsyncFileData) +        /* AsyncFileData structure */
     2 * pVfs->szOsFile +           /* AsyncFileData.pBaseRead and pBaseWrite */
@@ -739,30 +792,53 @@ static int asyncOpen(
     }
   }
 
+  pthread_mutex_lock(&async.lockMutex);
+
+  if( rc==SQLITE_OK ){
+    pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
+    if( !pLock ){
+      pLock = sqlite3MallocZero(pVfs->szOsFile + sizeof(AsyncLock));
+      if( pLock ){
+#ifdef ENABLE_FILE_LOCKING
+        if( flags&SQLITE_OPEN_MAIN_DB ){
+          pLock->pFile = (sqlite3_file *)&pLock[1];
+          rc = sqlite3OsOpen(pVfs, zName, pLock->pFile, flags, 0);
+          if( rc!=SQLITE_OK ){
+            sqlite3_free(pLock);
+            pLock = 0;
+          }
+        }
+#endif
+        sqlite3HashInsert(
+          &async.aLock, pData->zName, pData->nName, (void *)pLock
+        );
+      }else{
+        rc = SQLITE_NOMEM;
+      }
+    }
+  }
+
   if( rc==SQLITE_OK ){
     HashElem *pElem;
     p->pMethod = &async_methods;
     p->pData = pData;
     incrOpenFileCount();
 
-    /* Link AsyncFileData.lock into the linked list of AsyncLock structures
+    /* Link AsyncFileData.lock into the linked list of 
-    ** for this file. Obtain the async.lockMutex mutex before doing so.
+    ** AsyncFileLock structures for this file.
     */
-    AsyncLock *pNext;
+    pData->lock.pNext = pLock->pList;
-    pthread_mutex_lock(&async.lockMutex);
+    pLock->pList = &pData->lock;
-    pNext = sqlite3HashInsert(
+
-        &async.aLock, pData->zName, pData->nName, (void *)&pData->lock
-    );
-    pData->lock.pNext = pNext;
     pElem = sqlite3HashFindElem(&async.aLock, pData->zName, pData->nName);
     pData->zName = (char *)sqliteHashKey(pElem);
-    pthread_mutex_unlock(&async.lockMutex);
   }else{
     sqlite3OsClose(pData->pBaseRead);
     sqlite3OsClose(pData->pBaseWrite);
     sqlite3_free(pData);
   }
 
+  pthread_mutex_unlock(&async.lockMutex);
   return rc;
 }
 
@@ -1059,33 +1135,56 @@ static void *asyncWriterThread(void *NotUsed){
 
       case ASYNC_CLOSE: {
         AsyncLock *pLock;
+        AsyncFileLock **ppIter;
         AsyncFileData *pData = p->pFileData;
         ASYNC_TRACE(("CLOSE %s\n", p->pFileData->zName));
         sqlite3OsClose(pData->pBaseWrite);
         sqlite3OsClose(pData->pBaseRead);
 
-        /* Unlink AsyncFileData.lock from the linked list of AsyncLock 
+        /* Unlink AsyncFileData.lock from the linked list of AsyncFileLock 
         ** structures for this file. Obtain the async.lockMutex mutex 
         ** before doing so.
         */
         pthread_mutex_lock(&async.lockMutex);
         pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
-        if( pLock==&pData->lock ){
+        for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){
-          sqlite3HashInsert(
+          if( (*ppIter)==&pData->lock ){
-              &async.aLock, pData->zName, pData->nName, (void *)pLock->pNext 
+            *ppIter = (*ppIter)->pNext;
-          );
+            break;
-        }else{
-          for( ; pLock && pLock->pNext!=&pData->lock; pLock=pLock->pNext);
-          if( pLock ){
-            pLock->pNext = pData->lock.pNext;
           }
         }
+        if( !pLock->pList ){
+          if( pLock->pFile ) sqlite3OsClose(pLock->pFile);
+          sqlite3_free(pLock);
+          sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0);
+        }else{
+          rc = getFileLock(pLock);
+        }
         pthread_mutex_unlock(&async.lockMutex);
 
         sqlite3_free(pData);
         break;
       }
 
+      case ASYNC_UNLOCK: {
+        AsyncLock *pLock;
+        AsyncFileData *pData = p->pFileData;
+        int eLock = p->nByte;
+        pthread_mutex_lock(&async.lockMutex);
+        if( pData->lock.eAsyncLock>eLock ){
+          if( pData->lock.eLock>eLock ){
+            pData->lock.eAsyncLock = pData->lock.eLock;
+          }else{
+            pData->lock.eAsyncLock = eLock;
+          }
+        }
+        assert(pData->lock.eAsyncLock>=pData->lock.eLock);
+        pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName);
+        rc = getFileLock(pLock);
+        pthread_mutex_unlock(&async.lockMutex);
+        break;
+      }
+
       case ASYNC_DELETE:
         ASYNC_TRACE(("DELETE %s\n", p->zBuf));
         rc = sqlite3OsDelete(pVfs, p->zBuf, (int)p->iOffset);

test/async.test

@@ -6,7 +6,7 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: async.test,v 1.8 2007/09/04 14:31:47 danielk1977 Exp $
+# $Id: async.test,v 1.9 2007/09/04 18:28:44 danielk1977 Exp $
 
 
 if {[catch {sqlite3async_enable}]} {
@@ -32,7 +32,9 @@ set INCLUDE {
   trans.test
   lock.test
   lock3.test
+  lock2.test
 }
+# set INCLUDE lock4.test
 
 # Enable asynchronous IO.
 sqlite3async_enable 1