diff --git a/manifest b/manifest index 0a3424a6f3..e13e704a1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sdocumentation\son\sthe\sreturn\svalue\sfrom\ssqlite3_column_blob()\sfor\na\szero-length\sBLOB.\s\sClarify\sthe\sdocumentation\son\swhat\shappens\swhen\syou\nhave\sa\szeroblob()\swith\sa\snegative\slength.\s\sAdditional\stest\scases\sbut\sno\nchanges\sto\scode.\s\sTicket\s#2623.\s(CVS\s4395) -D 2007-09-04T12:18:42 +C Add\sinternal\slocking\sto\sthe\stest_async.c\sbackend.\sSo\sthat\smore\sthan\sone\sconnection\smay\sbe\sused\sfrom\swithin\sa\ssingle\sprocess.\s(CVS\s4396) +D 2007-09-04T14:31:47 F Makefile.in cbfb898945536a8f9ea8b897e1586dd1fdbcc5db F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -92,8 +92,8 @@ F src/delete.c 849846d06d29851dde0d9f424a5de5817eb140d1 F src/experimental.c 1b2d1a6cd62ecc39610e97670332ca073c50792b F src/expr.c 7853a8161ec0b0fce62fc8da921db557899f1ec1 F src/func.c 9d88141c4cffb3a04719e5a0fda65cde34bfa1e5 -F src/hash.c 06c69a3a038b713c43353c79023372bcfe7f5180 -F src/hash.h 3ad3da76bfb954978d227bf495568b0e6da2c19e +F src/hash.c 45a7005aac044b6c86bd7e49c44bc15d30006d6c +F src/hash.h 031cd9f915aff27e12262cb9eb570ac1b8326b53 F src/insert.c df9712e1f67201573a9677d3a2fe401d52d84dda F src/journal.c a45147d798f4d8dbdeed200ca7f740579bd8591b F src/legacy.c 4ac53191fad2e3c4d59bde1228879b2dc5a96d66 @@ -145,7 +145,7 @@ F src/test6.c 0513982dfef4da2a4154b538d2bf538b84ca21d3 F src/test7.c a9d509d0e9ad214b4772696f49f6e61be26213d1 F src/test8.c f113aa3723a52113d0fa7c28155ecd37e7e04077 F src/test9.c b46c8fe02ac7cca1a7316436d8d38d50c66f4b2f -F src/test_async.c dcb562dc81bf6851294b9260c8b61971169d5707 +F src/test_async.c c6f5f75f8ccf5d8be1076c45a9c3d5fd995249b9 F src/test_autoext.c 855157d97aa28cf84233847548bfacda21807436 F src/test_btree.c c1308ba0b88ab577fa56c9e493a09829dfcded9c F src/test_config.c 6fb459214b27952b143f45e35200d94096d54cc6 @@ -180,7 +180,7 @@ F test/alter2.test 50c3f554b8236d179d72511c0a4f23c5eb7f2af3 F test/alter3.test a6eec8f454be9b6ce73d8d7dc711453675a10ce7 F test/altermalloc.test 1f4d2d66750bea1a78cd9f0b7dba5bfb155dd6cf F test/analyze.test 2f55535aa335785db1a2f97d3f3831c16c09f8b0 -F test/async.test 464dc7c7ccb144e8c82ecca429e6d7cd1c96bd6e +F test/async.test b5547f56b329fa2ea5bce80f25c1d91eff36b47b F test/async2.test a8ef7abfda880b171b2f0a8476300816e33a808a F test/attach.test b849e1baae863c3a6132ff8b9b1baf356ab6c178 F test/attach2.test 78bc1a25ea8785c7571b44f5947ada2bd5d78127 @@ -569,7 +569,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 2eadef90162590a7b947c38acf0016d0c55821c7 -R a5be8e3038cc725b3d325f8649b6f5b5 -U drh -Z 9f6f2dde9339d174ebf4469c17340052 +P 63ca02a5b2700858f0eceadc9b58b942d473b191 +R 5cb983fe15fecf887cd0939edebbb838 +U danielk1977 +Z 34bdd4fc38cb5e365ec1bd9364840893 diff --git a/manifest.uuid b/manifest.uuid index e5e64f7833..bab485086e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ca02a5b2700858f0eceadc9b58b942d473b191 \ No newline at end of file +17ca684c124445f17d1e36c37e169056c5fd4569 \ No newline at end of file diff --git a/src/hash.c b/src/hash.c index d01ab92f72..a88d16b06b 100644 --- a/src/hash.c +++ b/src/hash.c @@ -12,7 +12,7 @@ ** This is the implementation of generic hash-tables ** used in SQLite. ** -** $Id: hash.c,v 1.23 2007/09/03 15:03:21 danielk1977 Exp $ +** $Id: hash.c,v 1.24 2007/09/04 14:31:47 danielk1977 Exp $ */ #include "sqliteInt.h" #include @@ -310,10 +310,11 @@ static void removeElementGivenHash( } /* Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is -** found, or NULL if there is no match. +** that matches pKey,nKey. Return a pointer to the corresponding +** HashElem structure for this element if it is found, or NULL +** otherwise. */ -void *sqlite3HashFind(const Hash *pH, const void *pKey, int nKey){ +HashElem *sqlite3HashFindElem(const Hash *pH, const void *pKey, int nKey){ int h; /* A hash on key */ HashElem *elem; /* The element that matches key */ int (*xHash)(const void*,int); /* The hash function */ @@ -324,6 +325,16 @@ void *sqlite3HashFind(const Hash *pH, const void *pKey, int nKey){ h = (*xHash)(pKey,nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1)); + return elem; +} + +/* Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +void *sqlite3HashFind(const Hash *pH, const void *pKey, int nKey){ + HashElem *elem; /* The element that matches key */ + elem = sqlite3HashFindElem(pH, pKey, nKey); return elem ? elem->data : 0; } diff --git a/src/hash.h b/src/hash.h index 275ac75454..e3274e9e44 100644 --- a/src/hash.h +++ b/src/hash.h @@ -12,7 +12,7 @@ ** This is the header file for the generic hash-table implemenation ** used in SQLite. ** -** $Id: hash.h,v 1.10 2007/08/16 04:30:40 drh Exp $ +** $Id: hash.h,v 1.11 2007/09/04 14:31:47 danielk1977 Exp $ */ #ifndef _SQLITE_HASH_H_ #define _SQLITE_HASH_H_ @@ -81,6 +81,7 @@ struct HashElem { void sqlite3HashInit(Hash*, int keytype, int copyKey); void *sqlite3HashInsert(Hash*, const void *pKey, int nKey, void *pData); void *sqlite3HashFind(const Hash*, const void *pKey, int nKey); +HashElem *sqlite3HashFindElem(const Hash*, const void *pKey, int nKey); void sqlite3HashClear(Hash*); /* diff --git a/src/test_async.c b/src/test_async.c index 7e382fbd5b..77c676b4a8 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -151,7 +151,7 @@ static void asyncTrace(const char *zFormat, ...){ ** queue in mid operation. ** ** -** asyncLock, asyncUnlock, asyncLockState, asyncCheckReservedLock +** asyncLock, asyncUnlock, asyncCheckReservedLock ** ** These primitives implement in-process locking using a hash table ** on the file name. Files are locked correctly for connections coming @@ -252,18 +252,14 @@ static struct TestAsyncStaticData { #define ASYNC_SYNC 2 #define ASYNC_TRUNCATE 3 #define ASYNC_CLOSE 4 -#define ASYNC_OPENDIRECTORY 5 -#define ASYNC_SETFULLSYNC 6 -#define ASYNC_DELETE 7 -#define ASYNC_OPENEXCLUSIVE 8 -#define ASYNC_SYNCDIRECTORY 9 +#define ASYNC_DELETE 5 +#define ASYNC_OPENEXCLUSIVE 6 /* Names of opcodes. Used for debugging only. ** Make sure these stay in sync with the macros above! */ static const char *azOpcodeName[] = { - "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", - "OPENDIR", "SETFULLSYNC", "DELETE", "OPENEX", "SYNCDIR", + "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX" }; /* @@ -273,6 +269,9 @@ static const char *azOpcodeName[] = { ** The interpretation of the iOffset and nByte variables varies depending ** on the value of AsyncWrite.op: ** +** ASYNC_NOOP: +** No values used. +** ** ASYNC_WRITE: ** iOffset -> Offset in file to write to. ** nByte -> Number of bytes of data to write (pointed to by zBuf). @@ -288,15 +287,6 @@ static const char *azOpcodeName[] = { ** iOffset -> Unused. ** nByte -> Unused. ** -** ASYNC_OPENDIRECTORY: -** iOffset -> Unused. -** nByte -> Number of bytes of zBuf points to (directory name). -** -** ASYNC_SETFULLSYNC: -** iOffset -> Unused. -** nByte -> New value for the full-sync flag. -** -** ** ASYNC_DELETE: ** iOffset -> Contains the "syncDir" flag. ** nByte -> Number of bytes of zBuf points to (file name). @@ -320,6 +310,23 @@ struct AsyncWrite { AsyncWrite *pNext; /* Next write operation (to any file) */ }; +/* +** An instance of the following structure is allocated along with each +** AsyncFileData structure (see AsyncFileData.lock), but is only used if the +** file was opened with the SQLITE_OPEN_MAIN_DB. +** +** The global async.aLock[] hash table maps from database file-name to a +** linked-list of AsyncLock structures corresponding to handles opened on the +** file. The AsyncLock structures are linked into the list when the file is +** opened and removed when it is closed. Mutex async.lockMutex must be held +** before accessing any AsyncLock structure or the async.aLock[] table. +*/ +typedef struct AsyncLock AsyncLock; +struct AsyncLock { + int eLock; + AsyncLock *pNext; +}; + /* ** The AsyncFile structure is a subclass of sqlite3_file used for ** asynchronous IO. @@ -340,6 +347,7 @@ struct AsyncFileData { int nName; /* Number of characters in zName */ sqlite3_file *pBaseRead; /* Read handle to the underlying Os file */ sqlite3_file *pBaseWrite; /* Write handle to the underlying Os file */ + AsyncLock lock; }; /* @@ -436,6 +444,12 @@ static int addNewAsyncWrite( */ static int asyncClose(sqlite3_file *pFile){ AsyncFileData *p = ((AsyncFile *)pFile)->pData; + + /* Unlock the file, if it is locked */ + pthread_mutex_lock(&async.lockMutex); + p->lock.eLock = 0; + pthread_mutex_unlock(&async.lockMutex); + return addNewAsyncWrite(p, ASYNC_CLOSE, 0, 0, 0); } @@ -488,9 +502,10 @@ static int asyncRead(sqlite3_file *pFile, void *zOut, int iAmt, i64 iOffset){ if( rc==SQLITE_OK ){ AsyncWrite *pWrite; + char *zName = p->zName; for(pWrite=async.pQueueFirst; pWrite; pWrite = pWrite->pNext){ - if( pWrite->pFileData==p && pWrite->op==ASYNC_WRITE ){ + if( pWrite->op==ASYNC_WRITE && pWrite->pFileData->zName==zName ){ int iBeginOut = (pWrite->iOffset-iOffset); int iBeginIn = -iBeginOut; int nCopy; @@ -558,7 +573,9 @@ int asyncFileSize(sqlite3_file *pFile, i64 *piSize){ if( rc==SQLITE_OK ){ AsyncWrite *pWrite; for(pWrite=async.pQueueFirst; pWrite; pWrite = pWrite->pNext){ - if( pWrite->pFileData==p ){ + if( pWrite->op==ASYNC_DELETE && strcmp(p->zName, pWrite->zBuf)==0 ){ + s = 0; + }else if( pWrite->pFileData && pWrite->pFileData->zName==p->zName){ switch( pWrite->op ){ case ASYNC_WRITE: s = MAX(pWrite->iOffset + (i64)(pWrite->nByte), s); @@ -583,30 +600,65 @@ int asyncFileSize(sqlite3_file *pFile, i64 *piSize){ ** cannot see our internal hash table (obviously) and will thus not ** honor our locks. */ -static int asyncLock(sqlite3_file *pFile, int lockType){ +static int asyncLock(sqlite3_file *pFile, int eLock){ + int rc = SQLITE_OK; AsyncFileData *p = ((AsyncFile *)pFile)->pData; - ASYNC_TRACE(("LOCK %d (%s)\n", lockType, p->zName)); + pthread_mutex_lock(&async.lockMutex); - sqlite3HashInsert(&async.aLock, p->zName, p->nName, (void*)lockType); + if( p->lock.eLockzName, p->nName); + assert(pLock); + for(/*no-op*/; pLock; pLock=pLock->pNext){ + if( pLock!=&p->lock && ( + (eLock==SQLITE_LOCK_EXCLUSIVE && pLock->eLock>=SQLITE_LOCK_SHARED) || + (eLock==SQLITE_LOCK_PENDING && pLock->eLock>=SQLITE_LOCK_RESERVED) || + (eLock==SQLITE_LOCK_RESERVED && pLock->eLock>=SQLITE_LOCK_RESERVED) || + (eLock==SQLITE_LOCK_SHARED && pLock->eLock>=SQLITE_LOCK_PENDING) + )){ + rc = SQLITE_BUSY; + } + } + if( rc==SQLITE_OK ){ + p->lock.eLock = eLock; + } + } + pthread_mutex_unlock(&async.lockMutex); + + ASYNC_TRACE(("LOCK %d (%s) rc=%d\n", eLock, p->zName, rc)); + return rc; +} +static int asyncUnlock(sqlite3_file *pFile, int eLock){ + AsyncFileData *p = ((AsyncFile *)pFile)->pData; + AsyncLock *pLock = &p->lock; + pthread_mutex_lock(&async.lockMutex); + if( pLock->eLock>eLock ){ + pLock->eLock = eLock; + } pthread_mutex_unlock(&async.lockMutex); return SQLITE_OK; } -static int asyncUnlock(sqlite3_file *pFile, int lockType){ - return asyncLock(pFile, lockType); -} /* ** This function is called when the pager layer first opens a database file ** and is checking for a hot-journal. */ static int asyncCheckReservedLock(sqlite3_file *pFile){ + int ret = 0; + AsyncLock *pLock; AsyncFileData *p = ((AsyncFile *)pFile)->pData; - int rc; + pthread_mutex_lock(&async.lockMutex); - rc = (int)sqlite3HashFind(&async.aLock, p->zName, p->nName); + pLock = (AsyncLock *)sqlite3HashFind(&async.aLock, p->zName, p->nName); + for(/*no-op*/; pLock; pLock=pLock->pNext){ + if( pLock->eLock>=SQLITE_LOCK_RESERVED ){ + ret = 1; + } + } pthread_mutex_unlock(&async.lockMutex); - ASYNC_TRACE(("CHECK-LOCK %d (%s)\n", rc, p->zName)); - return rc>SHARED_LOCK; + + ASYNC_TRACE(("CHECK-LOCK %d (%s)\n", ret, p->zName)); + return ret; } /* @@ -656,16 +708,16 @@ static int asyncOpen( sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; AsyncFile *p = (AsyncFile *)pFile; - int nName = strlen(zName);; + int nName = strlen(zName)+1; int rc; int nByte; AsyncFileData *pData; nByte = ( sizeof(AsyncFileData) + /* AsyncFileData structure */ - 2 * pVfs->szOsFile + /* AsyncFileData.zName */ - nName + 1 /* AsyncFileData.pBaseRead and pBaseWrite */ - ); + 2 * pVfs->szOsFile + /* AsyncFileData.pBaseRead and pBaseWrite */ + nName /* AsyncFileData.zName */ + ); pData = sqlite3_malloc(nByte); if( !pData ){ return SQLITE_NOMEM; @@ -673,9 +725,9 @@ static int asyncOpen( memset(pData, 0, nByte); pData->zName = (char *)&pData[1]; pData->nName = nName; - pData->pBaseRead = (sqlite3_file *)&pData->zName[nName+1]; - pData->pBaseWrite = (sqlite3_file *)&pData->zName[nName+1+pVfs->szOsFile]; - memcpy(pData->zName, zName, nName+1); + pData->pBaseRead = (sqlite3_file *)&pData->zName[nName]; + pData->pBaseWrite = (sqlite3_file *)&pData->zName[nName+pVfs->szOsFile]; + memcpy(pData->zName, zName, nName); if( flags&SQLITE_OPEN_EXCLUSIVE ){ rc = addNewAsyncWrite(pData, ASYNC_OPENEXCLUSIVE, (i64)flags, 0, 0); @@ -688,9 +740,23 @@ static int asyncOpen( } if( rc==SQLITE_OK ){ + HashElem *pElem; p->pMethod = &async_methods; p->pData = pData; incrOpenFileCount(); + + /* Link AsyncFileData.lock into the linked list of AsyncLock structures + ** for this file. Obtain the async.lockMutex mutex before doing so. + */ + AsyncLock *pNext; + pthread_mutex_lock(&async.lockMutex); + pNext = sqlite3HashInsert( + &async.aLock, pData->zName, pData->nName, (void *)&pData->lock + ); + pData->lock.pNext = pNext; + pElem = sqlite3HashFindElem(&async.aLock, pData->zName, pData->nName); + pData->zName = (char *)sqliteHashKey(pElem); + pthread_mutex_unlock(&async.lockMutex); }else{ sqlite3OsClose(pData->pBaseRead); sqlite3OsClose(pData->pBaseWrite); @@ -748,13 +814,61 @@ static int asyncGetTempName(sqlite3_vfs *pAsyncVfs, char *zBufOut){ sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; return pVfs->xGetTempName(pVfs, zBufOut); } + +/* +** Fill in zPathOut with the full path to the file identified by zPath. +*/ static int asyncFullPathname( sqlite3_vfs *pAsyncVfs, const char *zPath, char *zPathOut ){ + int rc; sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return sqlite3OsFullPathname(pVfs, zPath, zPathOut); + rc = sqlite3OsFullPathname(pVfs, zPath, zPathOut); + + /* Because of the way intra-process file locking works, this backend + ** needs to return a canonical path. The following block assumes the + ** file-system uses unix style paths. + */ + if( rc==SQLITE_OK ){ + int iIn; + int iOut = 0; + int nPathOut = strlen(zPathOut); + + for(iIn=0; iIn/../" with "" */ + if( iOut>0 && iIn<=(nPathOut-4) + && zPathOut[iIn]=='/' && zPathOut[iIn+1]=='.' + && zPathOut[iIn+2]=='.' && zPathOut[iIn+2]=='/' + ){ + iIn += 3; + iOut--; + for( ; iOut>0 && zPathOut[iOut]!='/'; iOut--); + continue; + } + + zPathOut[iOut++] = zPathOut[iIn]; + } + zPathOut[iOut] = '\0'; + } + + return rc; } static void *asyncDlOpen(sqlite3_vfs *pAsyncVfs, const char *zPath){ sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; @@ -943,12 +1057,34 @@ static void *asyncWriterThread(void *NotUsed){ rc = sqlite3OsTruncate(pBase, p->iOffset); break; - case ASYNC_CLOSE: + case ASYNC_CLOSE: { + AsyncLock *pLock; + AsyncFileData *pData = p->pFileData; ASYNC_TRACE(("CLOSE %s\n", p->pFileData->zName)); - sqlite3OsClose(p->pFileData->pBaseWrite); - sqlite3OsClose(p->pFileData->pBaseRead); - sqlite3_free(p->pFileData); + sqlite3OsClose(pData->pBaseWrite); + sqlite3OsClose(pData->pBaseRead); + + /* Unlink AsyncFileData.lock from the linked list of AsyncLock + ** structures for this file. Obtain the async.lockMutex mutex + ** before doing so. + */ + pthread_mutex_lock(&async.lockMutex); + pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); + if( pLock==&pData->lock ){ + sqlite3HashInsert( + &async.aLock, pData->zName, pData->nName, (void *)pLock->pNext + ); + }else{ + for( ; pLock && pLock->pNext!=&pData->lock; pLock=pLock->pNext); + if( pLock ){ + pLock->pNext = pData->lock.pNext; + } + } + pthread_mutex_unlock(&async.lockMutex); + + sqlite3_free(pData); break; + } case ASYNC_DELETE: ASYNC_TRACE(("DELETE %s\n", p->zBuf)); diff --git a/test/async.test b/test/async.test index 2a723be975..08dbb7b57f 100644 --- a/test/async.test +++ b/test/async.test @@ -6,7 +6,7 @@ #*********************************************************************** # This file runs all tests. # -# $Id: async.test,v 1.7 2006/03/19 13:00:25 drh Exp $ +# $Id: async.test,v 1.8 2007/09/04 14:31:47 danielk1977 Exp $ if {[catch {sqlite3async_enable}]} { @@ -30,8 +30,9 @@ set INCLUDE { insert2.test insert3.test trans.test + lock.test + lock3.test } -# set INCLUDE {select4.test} # Enable asynchronous IO. sqlite3async_enable 1