diff --git a/manifest b/manifest index f1add3dcf5..e5016616b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhereby\sthe\s*ppVtab\soutput\sbuffer\spassed\sto\ssqlite3_module.xConstruct()\scould\sbe\sinvalidated\s(freed)\sif\sa\smalloc()\sfailure\soccured\swithin\sa\scall\sto\ssqlite3_declare_vtab().\s(CVS\s4397) -D 2007-09-04T15:38:58 +C Add\sexternal\slocking\sto\stest_async.c.\sThere\sare\sstill\ssome\stests\sto\scome.\s(CVS\s4398) +D 2007-09-04T18:28:44 F Makefile.in cbfb898945536a8f9ea8b897e1586dd1fdbcc5db F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -145,7 +145,7 @@ F src/test6.c 0513982dfef4da2a4154b538d2bf538b84ca21d3 F src/test7.c a9d509d0e9ad214b4772696f49f6e61be26213d1 F src/test8.c f113aa3723a52113d0fa7c28155ecd37e7e04077 F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f -F src/test_async.c c6f5f75f8ccf5d8be1076c45a9c3d5fd995249b9 +F src/test_async.c e221db3e87b472733a8015be7d70bae0edb848b1 F src/test_autoext.c 855157d97aa28cf84233847548bfacda21807436 F src/test_btree.c c1308ba0b88ab577fa56c9e493a09829dfcded9c F src/test_config.c 6fb459214b27952b143f45e35200d94096d54cc6 @@ -180,7 +180,7 @@ F test/alter2.test 50c3f554b8236d179d72511c0a4f23c5eb7f2af3 F test/alter3.test a6eec8f454be9b6ce73d8d7dc711453675a10ce7 F test/altermalloc.test 1f4d2d66750bea1a78cd9f0b7dba5bfb155dd6cf F test/analyze.test 2f55535aa335785db1a2f97d3f3831c16c09f8b0 -F test/async.test b5547f56b329fa2ea5bce80f25c1d91eff36b47b +F test/async.test 18e7dc66535f3d86c05e0f954384472e2ed52490 F test/async2.test a8ef7abfda880b171b2f0a8476300816e33a808a F test/attach.test b849e1baae863c3a6132ff8b9b1baf356ab6c178 F test/attach2.test 78bc1a25ea8785c7571b44f5947ada2bd5d78127 @@ -569,7 +569,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 17ca684c124445f17d1e36c37e169056c5fd4569 -R aab5e4f24ec35c5749ccbfc9ee626576 +P efd61df1b9170f0134787ae17ac996a7eff64add +R ef7fa5443cd985d720e63f8ac953cec0 U danielk1977 -Z d2e7e7e0b743acfcb2048b0ee18425ea +Z 605a822a91549091d299d216a22bbd33 diff --git a/manifest.uuid b/manifest.uuid index 0d17899a1f..a63f7e2ede 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efd61df1b9170f0134787ae17ac996a7eff64add \ No newline at end of file +3794dcd31a74e90b181b336bf6a4c917bda526b8 \ No newline at end of file diff --git a/src/test_async.c b/src/test_async.c index 77c676b4a8..e03eda2c20 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -70,6 +70,13 @@ ** when the buffer gets to be too big. */ +/* +** If this symbol is defined, then file-system locks are obtained as +** required. This slows things down, but allows multiple processes +** to access the database concurrently. +*/ +#define ENABLE_FILE_LOCKING + #include "sqliteInt.h" #include @@ -79,6 +86,7 @@ */ #if OS_UNIX && SQLITE_THREADSAFE + /* ** This demo uses pthreads. If you do not have a pthreads implementation ** for your operating system, you will need to recode the threading @@ -95,6 +103,8 @@ typedef struct AsyncWrite AsyncWrite; typedef struct AsyncFile AsyncFile; typedef struct AsyncFileData AsyncFileData; +typedef struct AsyncFileLock AsyncFileLock; +typedef struct AsyncLock AsyncLock; /* Enable for debugging */ static int sqlite3async_trace = 0; @@ -132,8 +142,7 @@ static void asyncTrace(const char *zFormat, ...){ ** ** File handle operations (invoked by SQLite thread): ** -** asyncWrite, asyncClose, asyncTruncate, asyncSync, -** asyncSetFullSync, asyncOpenDirectory. +** asyncWrite, asyncClose, asyncTruncate, asyncSync ** ** The operations above add an entry to the global write-op list. They ** prepare the entry, acquire the async.queueMutex momentarily while @@ -159,23 +168,6 @@ static void asyncTrace(const char *zFormat, ...){ ** and will therefore not honor them. ** ** -** asyncFileHandle. -** -** The sqlite3OsFileHandle() function is currently only used when -** debugging the pager module. Unless sqlite3OsClose() is called on the -** file (shouldn't be possible for other reasons), the underlying -** implementations are safe to call without grabbing any mutex. So we just -** go ahead and call it no matter what any other threads are doing. -** -** -** asyncSeek. -** -** Calling this method just manipulates the AsyncFile.iOffset variable. -** Since this variable is never accessed by writer thread, this -** function does not require the mutex. Actual calls to OsSeek() take -** place just before OsWrite() or OsRead(), which are always protected by -** the mutex. -** ** The writer thread: ** ** The async.writerMutex is used to make sure only there is only @@ -222,7 +214,9 @@ static void asyncTrace(const char *zFormat, ...){ /* ** State information is held in the static variable "async" defined -** as follows: +** as the following structure. +** +** Both async.ioError and async.nFile are protected by async.queueMutex. */ static struct TestAsyncStaticData { pthread_mutex_t queueMutex; /* Mutex for access to write operation queue */ @@ -254,12 +248,13 @@ static struct TestAsyncStaticData { #define ASYNC_CLOSE 4 #define ASYNC_DELETE 5 #define ASYNC_OPENEXCLUSIVE 6 +#define ASYNC_UNLOCK 7 /* Names of opcodes. Used for debugging only. ** Make sure these stay in sync with the macros above! */ static const char *azOpcodeName[] = { - "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX" + "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX", "UNLOCK" }; /* @@ -295,6 +290,9 @@ static const char *azOpcodeName[] = { ** iOffset -> Value of "delflag". ** nByte -> Number of bytes of zBuf points to (file name). ** +** ASYNC_UNLOCK: +** nByte -> Argument to sqlite3OsUnlock(). +** ** ** For an ASYNC_WRITE operation, zBuf points to the data to write to the file. ** This space is sqlite3_malloc()d along with the AsyncWrite structure in a @@ -316,15 +314,22 @@ struct AsyncWrite { ** file was opened with the SQLITE_OPEN_MAIN_DB. ** ** The global async.aLock[] hash table maps from database file-name to a -** linked-list of AsyncLock structures corresponding to handles opened on the -** file. The AsyncLock structures are linked into the list when the file is -** opened and removed when it is closed. Mutex async.lockMutex must be held -** before accessing any AsyncLock structure or the async.aLock[] table. +** linked-list of AsyncFileLock structures corresponding to handles opened on +** the file. The AsyncFileLock structures are linked into the list when the +** file is opened and removed when it is closed. Mutex async.lockMutex must be +** held before accessing any AsyncFileLock structure or the async.aLock[] +** table. */ -typedef struct AsyncLock AsyncLock; +struct AsyncFileLock { + int eLock; /* Internally visible lock state (sqlite pov) */ + int eAsyncLock; /* Lock-state with write-queue unlock */ + AsyncFileLock *pNext; +}; + struct AsyncLock { + sqlite3_file *pFile; int eLock; - AsyncLock *pNext; + AsyncFileLock *pList; }; /* @@ -347,7 +352,7 @@ struct AsyncFileData { int nName; /* Number of characters in zName */ sqlite3_file *pBaseRead; /* Read handle to the underlying Os file */ sqlite3_file *pBaseWrite; /* Write handle to the underlying Os file */ - AsyncLock lock; + AsyncFileLock lock; }; /* @@ -478,16 +483,17 @@ static int asyncRead(sqlite3_file *pFile, void *zOut, int iAmt, i64 iOffset){ int nRead; sqlite3_file *pBase = p->pBaseRead; + /* Grab the write queue mutex for the duration of the call */ + pthread_mutex_lock(&async.queueMutex); + /* If an I/O error has previously occurred in this virtual file ** system, then all subsequent operations fail. */ if( async.ioError!=SQLITE_OK ){ - return async.ioError; + rc = async.ioError; + goto asyncread_out; } - /* Grab the write queue mutex for the duration of the call */ - pthread_mutex_lock(&async.queueMutex); - if( pBase->pMethods ){ rc = sqlite3OsFileSize(pBase, &filesize); if( rc!=SQLITE_OK ){ @@ -592,6 +598,34 @@ int asyncFileSize(sqlite3_file *pFile, i64 *piSize){ return rc; } +/* +** Lock or unlock the actual file-system entry. +*/ +static int getFileLock(AsyncLock *pLock){ + int rc = SQLITE_OK; + AsyncFileLock *pIter; + int eRequired = 0; + + if( pLock->pFile ){ + for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ + assert(pIter->eAsyncLock>=pIter->eLock); + if( pIter->eAsyncLock>eRequired ){ + eRequired = pIter->eAsyncLock; + } + } + if( eRequired>pLock->eLock ){ + rc = sqlite3OsLock(pLock->pFile, eRequired); + }else if(eRequiredeLock){ + rc = sqlite3OsUnlock(pLock->pFile, eRequired); + } + if( rc==SQLITE_OK ){ + pLock->eLock = eRequired; + } + } + + return rc; +} + /* ** No disk locking is performed. We keep track of locks locally in ** the async.aLock hash table. Locking should appear to work the same @@ -607,20 +641,28 @@ static int asyncLock(sqlite3_file *pFile, int eLock){ pthread_mutex_lock(&async.lockMutex); if( p->lock.eLockzName, p->nName); - assert(pLock); - for(/*no-op*/; pLock; pLock=pLock->pNext){ - if( pLock!=&p->lock && ( - (eLock==SQLITE_LOCK_EXCLUSIVE && pLock->eLock>=SQLITE_LOCK_SHARED) || - (eLock==SQLITE_LOCK_PENDING && pLock->eLock>=SQLITE_LOCK_RESERVED) || - (eLock==SQLITE_LOCK_RESERVED && pLock->eLock>=SQLITE_LOCK_RESERVED) || - (eLock==SQLITE_LOCK_SHARED && pLock->eLock>=SQLITE_LOCK_PENDING) + assert(pLock && pLock->pList); + for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ + if( pIter!=&p->lock && ( + (eLock==SQLITE_LOCK_EXCLUSIVE && pIter->eLock>=SQLITE_LOCK_SHARED) || + (eLock==SQLITE_LOCK_PENDING && pIter->eLock>=SQLITE_LOCK_RESERVED) || + (eLock==SQLITE_LOCK_RESERVED && pIter->eLock>=SQLITE_LOCK_RESERVED) || + (eLock==SQLITE_LOCK_SHARED && pIter->eLock>=SQLITE_LOCK_PENDING) )){ rc = SQLITE_BUSY; } } if( rc==SQLITE_OK ){ p->lock.eLock = eLock; + if( eLock>p->lock.eAsyncLock ){ + p->lock.eAsyncLock = eLock; + } + } + assert(p->lock.eAsyncLock>=p->lock.eLock); + if( rc==SQLITE_OK ){ + rc = getFileLock(pLock); } } pthread_mutex_unlock(&async.lockMutex); @@ -630,13 +672,13 @@ static int asyncLock(sqlite3_file *pFile, int eLock){ } static int asyncUnlock(sqlite3_file *pFile, int eLock){ AsyncFileData *p = ((AsyncFile *)pFile)->pData; - AsyncLock *pLock = &p->lock; + AsyncFileLock *pLock = &p->lock; pthread_mutex_lock(&async.lockMutex); if( pLock->eLock>eLock ){ pLock->eLock = eLock; } pthread_mutex_unlock(&async.lockMutex); - return SQLITE_OK; + return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0); } /* @@ -645,13 +687,14 @@ static int asyncUnlock(sqlite3_file *pFile, int eLock){ */ static int asyncCheckReservedLock(sqlite3_file *pFile){ int ret = 0; + AsyncFileLock *pIter; AsyncLock *pLock; AsyncFileData *p = ((AsyncFile *)pFile)->pData; pthread_mutex_lock(&async.lockMutex); pLock = (AsyncLock *)sqlite3HashFind(&async.aLock, p->zName, p->nName); - for(/*no-op*/; pLock; pLock=pLock->pNext){ - if( pLock->eLock>=SQLITE_LOCK_RESERVED ){ + for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ + if( pIter->eLock>=SQLITE_LOCK_RESERVED ){ ret = 1; } } @@ -665,6 +708,14 @@ static int asyncCheckReservedLock(sqlite3_file *pFile){ ** This is a no-op, as the asynchronous backend does not support locking. */ static int asyncFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + pthread_mutex_lock(&async.lockMutex); + *(int*)pArg = ((AsyncFile*)id)->pData->lock.eLock; + pthread_mutex_unlock(&async.lockMutex); + return SQLITE_OK; + } + } return SQLITE_ERROR; } @@ -713,6 +764,8 @@ static int asyncOpen( int nByte; AsyncFileData *pData; + AsyncLock *pLock = 0; + nByte = ( sizeof(AsyncFileData) + /* AsyncFileData structure */ 2 * pVfs->szOsFile + /* AsyncFileData.pBaseRead and pBaseWrite */ @@ -739,30 +792,53 @@ static int asyncOpen( } } + pthread_mutex_lock(&async.lockMutex); + + if( rc==SQLITE_OK ){ + pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); + if( !pLock ){ + pLock = sqlite3MallocZero(pVfs->szOsFile + sizeof(AsyncLock)); + if( pLock ){ +#ifdef ENABLE_FILE_LOCKING + if( flags&SQLITE_OPEN_MAIN_DB ){ + pLock->pFile = (sqlite3_file *)&pLock[1]; + rc = sqlite3OsOpen(pVfs, zName, pLock->pFile, flags, 0); + if( rc!=SQLITE_OK ){ + sqlite3_free(pLock); + pLock = 0; + } + } +#endif + sqlite3HashInsert( + &async.aLock, pData->zName, pData->nName, (void *)pLock + ); + }else{ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ HashElem *pElem; p->pMethod = &async_methods; p->pData = pData; incrOpenFileCount(); - /* Link AsyncFileData.lock into the linked list of AsyncLock structures - ** for this file. Obtain the async.lockMutex mutex before doing so. + /* Link AsyncFileData.lock into the linked list of + ** AsyncFileLock structures for this file. */ - AsyncLock *pNext; - pthread_mutex_lock(&async.lockMutex); - pNext = sqlite3HashInsert( - &async.aLock, pData->zName, pData->nName, (void *)&pData->lock - ); - pData->lock.pNext = pNext; + pData->lock.pNext = pLock->pList; + pLock->pList = &pData->lock; + pElem = sqlite3HashFindElem(&async.aLock, pData->zName, pData->nName); pData->zName = (char *)sqliteHashKey(pElem); - pthread_mutex_unlock(&async.lockMutex); }else{ sqlite3OsClose(pData->pBaseRead); sqlite3OsClose(pData->pBaseWrite); sqlite3_free(pData); } + pthread_mutex_unlock(&async.lockMutex); return rc; } @@ -1059,33 +1135,56 @@ static void *asyncWriterThread(void *NotUsed){ case ASYNC_CLOSE: { AsyncLock *pLock; + AsyncFileLock **ppIter; AsyncFileData *pData = p->pFileData; ASYNC_TRACE(("CLOSE %s\n", p->pFileData->zName)); sqlite3OsClose(pData->pBaseWrite); sqlite3OsClose(pData->pBaseRead); - /* Unlink AsyncFileData.lock from the linked list of AsyncLock + /* Unlink AsyncFileData.lock from the linked list of AsyncFileLock ** structures for this file. Obtain the async.lockMutex mutex ** before doing so. */ pthread_mutex_lock(&async.lockMutex); pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); - if( pLock==&pData->lock ){ - sqlite3HashInsert( - &async.aLock, pData->zName, pData->nName, (void *)pLock->pNext - ); - }else{ - for( ; pLock && pLock->pNext!=&pData->lock; pLock=pLock->pNext); - if( pLock ){ - pLock->pNext = pData->lock.pNext; + for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){ + if( (*ppIter)==&pData->lock ){ + *ppIter = (*ppIter)->pNext; + break; } } + if( !pLock->pList ){ + if( pLock->pFile ) sqlite3OsClose(pLock->pFile); + sqlite3_free(pLock); + sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0); + }else{ + rc = getFileLock(pLock); + } pthread_mutex_unlock(&async.lockMutex); sqlite3_free(pData); break; } + case ASYNC_UNLOCK: { + AsyncLock *pLock; + AsyncFileData *pData = p->pFileData; + int eLock = p->nByte; + pthread_mutex_lock(&async.lockMutex); + if( pData->lock.eAsyncLock>eLock ){ + if( pData->lock.eLock>eLock ){ + pData->lock.eAsyncLock = pData->lock.eLock; + }else{ + pData->lock.eAsyncLock = eLock; + } + } + assert(pData->lock.eAsyncLock>=pData->lock.eLock); + pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); + rc = getFileLock(pLock); + pthread_mutex_unlock(&async.lockMutex); + break; + } + case ASYNC_DELETE: ASYNC_TRACE(("DELETE %s\n", p->zBuf)); rc = sqlite3OsDelete(pVfs, p->zBuf, (int)p->iOffset); diff --git a/test/async.test b/test/async.test index 08dbb7b57f..8575555565 100644 --- a/test/async.test +++ b/test/async.test @@ -6,7 +6,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: async.test,v 1.8 2007/09/04 14:31:47 danielk1977 Exp $ +# $Id: async.test,v 1.9 2007/09/04 18:28:44 danielk1977 Exp $ if {[catch {sqlite3async_enable}]} { @@ -32,7 +32,9 @@ set INCLUDE { trans.test lock.test lock3.test + lock2.test } +# set INCLUDE lock4.test # Enable asynchronous IO. sqlite3async_enable 1