diff --git a/manifest b/manifest index 894b8432a6..9934b66edf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\sin\swal2savepoint.test. -D 2018-12-18T18:16:00.494 +C Add\sthe\s"mutexfree-shmlock"\spatch\sto\sthis\sbranch. +D 2018-12-18T19:46:50.625 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6 @@ -496,7 +496,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 1aa113b261a0ad44fd410a001f6e39bfc1ebd4279b2fb8c2d636a7620d76a45a +F src/os_unix.c 9886c03162c253d0845c40baef0f105bc540d53efc8378c2ef1598406c26d9be F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c bd11bf6971789994b4c1972019f16caf6e5ed5a59165d65f0147535aab191703 @@ -563,12 +563,12 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef -F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e +F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 F src/test_tclsh.c c225ebf05d8905e204580198ffaa09847256ac58af04af632fde3fd760475d90 F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 -F src/test_vfs.c 112f1f9271c33c211812e0e681830a84262dac065da58579ff49f9cefec97d4f +F src/test_vfs.c 194b8e78e9b4279bc69693cb5cfc343e0a08482c430a571ec96d80440e337a44 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215 @@ -594,7 +594,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c dca50fc1279f726751947aaa9334ad946094f55f6b0a850e70e58e41037f9433 +F src/wal.c da000d7e4e4702eb4d2cf7c08d734db8b522c3ac43350323a0df0968ba566925 F src/wal.h 4ef31f6defdd12877eff7832eb5f56c1f898bc074ac5aab67a539f9c59cb8118 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1094,7 +1094,7 @@ F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 -F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035 +F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -1196,7 +1196,7 @@ F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test d0f878173c36562b50898ecccc1a021f06dbb17c3b41ee5b6e620eb4c872a73c +F test/permutations.test c26601c1db80a67a54353af19a63a6ff5fbf8c80a2b2554e10e4f9030c428fee F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1300,6 +1300,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013 +F test/shmlock.test 3d1868f0386923c0592a235f2dd87ae52286b217bc695fbfd9d39a828e7be374 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1609,7 +1610,7 @@ F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca675 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad -F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 +F test/wal.test cdf0ca6cc0447520d19ef1c83287824ebeb3e82d75af856511ba96841a79fc9b F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b F test/wal2concurrent.test 7fc3e570073683a2a28f42bda46ecf516f5bc82afd703c1fbf4aa38e18fb3361 @@ -1812,7 +1813,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2c618c0a45e1b1d271a4a981ee1e3b368dfcce94eda5c2d7d08f794784046d9 -R 401d56567e7ccded06481190d60905d2 +P e388d02940e8c17e8e3f460643d988c8f523382df343f4fe766c2da17878e98d b9a741519055a8560ec1d0ab7d0c15ff8db53e740011825506cd2ede11c956f0 +R 9372febc05219e660f9e03e31a064ff1 U dan -Z fd40ed8c5ba115c1458cb1532edc718e +Z 52542d1d3ae1e576093729b891bf71b2 diff --git a/manifest.uuid b/manifest.uuid index b2ee1bd12e..37f165021c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e388d02940e8c17e8e3f460643d988c8f523382df343f4fe766c2da17878e98d \ No newline at end of file +c3191fc8fcb0775f87f55cb8124e0c36caa54993c2a5fe770d437e4f6ee72d77 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6c48ecfb0c..cb8e9e7131 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,16 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Turn this feature on in all builds for now */ +#define SQLITE_MUTEXFREE_SHMLOCK 1 +#define SQLITE_MFS_EXCLUSIVE 255 +#ifndef SQLITE_MFS_NSHARD +# define SQLITE_MFS_NSHARD 8 +#endif +#if SQLITE_MFS_NSHARD<1 +# error "SQLITE_MFS_NSHARD must be greater than 0" +#endif + /* ** There are various methods for file locking used for concurrency ** control: @@ -4261,8 +4271,41 @@ struct unixShmNode { u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif + +#ifdef SQLITE_MUTEXFREE_SHMLOCK + /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks + ** are stored in the following 64-bit value. There are in total 8 + ** shm-locking slots, each of which are assigned 8-bits from the 64-bit + ** value. The least-significant 8 bits correspond to shm-locking slot + ** 0, and so on. + ** + ** If the 8-bits corresponding to a shm-locking locking slot are set to + ** 0xFF, then a write-lock is held on the slot. Or, if they are set to + ** a non-zero value smaller than 0xFF, then they represent the total + ** number of read-locks held on the slot. There is no way to distinguish + ** between a write-lock and 255 read-locks. */ + struct LockingSlot { + u32 nLock; + u64 aPadding[7]; + } aMFSlot[3 + SQLITE_MFS_NSHARD*5]; +#endif }; +/* +** Atomic CAS primitive used in multi-process mode. Equivalent to: +** +** int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){ +** if( *ptr==oldval ){ +** *ptr = newval; +** return 1; +** } +** return 0; +** } +*/ +#define unixCompareAndSwap(ptr,oldval,newval) \ + __sync_bool_compare_and_swap(ptr,oldval,newval) + + /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. @@ -4283,6 +4326,9 @@ struct unixShm { u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_MUTEXFREE_SHMLOCK + u8 aMFCurrent[8]; /* Current slot used for each shared lock */ +#endif }; /* @@ -4787,6 +4833,83 @@ shmpage_out: return rc; } +#ifdef SQLITE_MUTEXFREE_SHMLOCK +static int unixMutexFreeShmlock( + unixFile *pFd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + struct LockMapEntry { + int iFirst; + int nSlot; + } aMap[9] = { + { 0, 1 }, + { 1, 1 }, + { 2, 1 }, + { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+5*SQLITE_MFS_NSHARD, 0 }, + }; + + unixShm *p = pFd->pShm; /* The shared memory being locked */ + unixShm *pX; /* For looping over all siblings */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; + int iIncr; + u16 mask; /* Mask of locks to take or release */ + + if( flags & SQLITE_SHM_SHARED ){ + /* SHARED locks */ + u32 iOld, iNew, *ptr; + int iIncr = -1; + if( (flags & SQLITE_SHM_UNLOCK)==0 ){ + p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot; + iIncr = 1; + } + ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock; + do { + iOld = *ptr; + iNew = iOld + iIncr; + if( iNew>SQLITE_MFS_EXCLUSIVE ){ + return SQLITE_BUSY; + } + }while( 0==unixCompareAndSwap(ptr, iOld, iNew) ); + }else{ + /* EXCLUSIVE locks */ + int iFirst = aMap[ofst].iFirst; + int iLast = aMap[ofst+n].iFirst; + int i; + for(i=iFirst; iaMFSlot[i].nLock; + if( flags & SQLITE_SHM_UNLOCK ){ + assert( (*ptr)==SQLITE_MFS_EXCLUSIVE ); + *ptr = 0; + }else{ + u32 iOld; + do { + iOld = *ptr; + if( iOld>0 ){ + while( i>iFirst ){ + i--; + pShmNode->aMFSlot[i].nLock = 0; + } + return SQLITE_BUSY; + } + }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) ); + } + } + } + + return SQLITE_OK; +} +#else +# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK +#endif + /* ** Change the lock state for a shared-memory segment. ** @@ -4820,8 +4943,20 @@ static int unixShmLock( assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + if( pDbFd->pInode->bProcessLock ){ + return unixMutexFreeShmlock(pDbFd, ofst, n, flags); + } + mask = (1<<(ofst+n)) - (1<1 || mask==(1<sharedMask&mask)==0 ); + assert( !(flags&SQLITE_SHM_EXCLUSIVE) || !(p->exclMask&mask) ); + }else{ + assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==mask ); + assert( !(flags&SQLITE_SHM_EXCLUSIVE) || (p->exclMask&mask)==mask ); + } + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ @@ -4911,12 +5046,16 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); +#ifdef SQLITE_MUTEXFREE_SHMLOCK + __sync_synchronize(); +#else sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ assert( fd->pMethods->xLock==nolockLock || unixFileMutexNotheld((unixFile*)fd) ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); +#endif } /* diff --git a/src/test_superlock.c b/src/test_superlock.c index 45d0d623a0..d677ce5378 100644 --- a/src/test_superlock.c +++ b/src/test_superlock.c @@ -41,6 +41,8 @@ typedef struct SuperlockBusy SuperlockBusy; struct Superlock { sqlite3 *db; /* Database handle used to lock db */ int bWal; /* True if db is a WAL database */ + int bRecoveryLocked; /* True if WAL RECOVERY lock is held */ + int bReaderLocked; /* True if WAL READER locks are held */ }; typedef struct Superlock Superlock; @@ -107,12 +109,13 @@ static int superlockShmLock( ** Invoke the supplied busy-handler as required. */ static int superlockWalLock( - sqlite3 *db, /* Database handle open on WAL database */ + Superlock *pLock, /* Superlock handle */ SuperlockBusy *pBusy /* Busy handler wrapper object */ ){ int rc; /* Return code */ sqlite3_file *fd = 0; /* Main database file handle */ void volatile *p = 0; /* Pointer to first page of shared memory */ + sqlite3 *db = pLock->db; /* Obtain a pointer to the sqlite3_file object open on the main db file. */ rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd); @@ -121,8 +124,10 @@ static int superlockWalLock( /* Obtain the "recovery" lock. Normally, this lock is only obtained by ** clients running database recovery. */ + assert( pLock->bRecoveryLocked==0 ); rc = superlockShmLock(fd, 2, 1, pBusy); if( rc!=SQLITE_OK ) return rc; + pLock->bRecoveryLocked = 1; /* Zero the start of the first shared-memory page. This means that any ** clients that open read or write transactions from this point on will @@ -139,7 +144,9 @@ static int superlockWalLock( ** are held, it is guaranteed that there are no active reader, writer or ** checkpointer clients. */ + assert( pLock->bReaderLocked==0 ); rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy); + if( rc==SQLITE_OK ) pLock->bReaderLocked = 1; return rc; } @@ -156,8 +163,14 @@ void sqlite3demo_superunlock(void *pLock){ sqlite3_file *fd = 0; rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd); if( rc==SQLITE_OK ){ - fd->pMethods->xShmLock(fd, 2, 1, flags); - fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags); + if( p->bRecoveryLocked ){ + fd->pMethods->xShmLock(fd, 2, 1, flags); + p->bRecoveryLocked = 0; + } + if( p->bReaderLocked ){ + fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags); + p->bReaderLocked = 0; + } } } sqlite3_close(p->db); @@ -232,7 +245,7 @@ int sqlite3demo_superlock( if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){ rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = superlockWalLock(pLock->db, &busy); + rc = superlockWalLock(pLock, &busy); } } } diff --git a/src/test_vfs.c b/src/test_vfs.c index 4a98ac214c..645b1447c2 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -1563,8 +1563,62 @@ static int SQLITE_TCLAPI testvfs_cmd( return TCL_ERROR; } +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern const char *sqlite3ErrName(int); + +/* +** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N +*/ +static int SQLITE_TCLAPI test_vfs_shmlock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *azArg1[] = {"shared", "exclusive", 0}; + const char *azArg2[] = {"lock", "unlock", 0}; + sqlite3 *db = 0; + int rc = SQLITE_OK; + const char *zDbname = 0; + int iArg1 = 0; + int iArg2 = 0; + int iOffset = 0; + int n = 0; + sqlite3_file *pFd; + + if( objc!=7 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N" + ); + return TCL_ERROR; + } + + zDbname = Tcl_GetString(objv[2]); + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) + || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) + || Tcl_GetIntFromObj(interp, objv[5], &iOffset) + || Tcl_GetIntFromObj(interp, objv[6], &n) + ){ + return TCL_ERROR; + } + + sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + if( pFd==0 ){ + return TCL_ERROR; + } + rc = pFd->pMethods->xShmLock(pFd, iOffset, n, + (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE) + | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_OK; +} + + int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); + Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0); return TCL_OK; } diff --git a/src/wal.c b/src/wal.c index 5e9ca3691f..426ba3cbb1 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3412,19 +3412,14 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); - /* It is possible that there is a checkpointer thread running - ** concurrent with this code. If this is the case, it may be that the - ** checkpointer has already determined that it will checkpoint + /* If it were possible for a checkpointer thread to run concurrent + ** with this code, it would be a problem. In this case, it could be + ** that the checkpointer has already determined that it will checkpoint ** snapshot X, where X is later in the wal file than pSnapshot, but ** has not yet set the pInfo->nBackfillAttempted variable to indicate - ** its intent. To avoid the race condition this leads to, ensure that - ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. - ** - ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing - ** this already? - */ - rc = walLockShared(pWal, WAL_CKPT_LOCK); + ** its intent. Fortunately this is not possible, as the call to + ** sqlite3WalSnapshotOpen() that sets pWal->pSnapshot also takes a + ** SHARED lock on the checkpointer slot. */ if( rc==SQLITE_OK ){ /* Check that the wal file has not been wrapped. Assuming that it has @@ -3443,8 +3438,6 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ rc = SQLITE_ERROR_SNAPSHOT; } - /* Release the shared CKPT lock obtained above. */ - walUnlockShared(pWal, WAL_CKPT_LOCK); pWal->minFrame = 1; } diff --git a/test/lock_common.tcl b/test/lock_common.tcl index a758e7af2e..3e1821bab0 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -15,18 +15,20 @@ proc do_multiclient_test {varname script} { - foreach code [list { + foreach {tn code} [list 1 { if {[info exists ::G(valgrind)]} { db close ; continue } set ::code2_chan [launch_testfixture] set ::code3_chan [launch_testfixture] proc code2 {tcl} { testfixture $::code2_chan $tcl } proc code3 {tcl} { testfixture $::code3_chan $tcl } - set tn 1 - } { + } 2 { proc code2 {tcl} { uplevel #0 $tcl } proc code3 {tcl} { uplevel #0 $tcl } - set tn 2 }] { + # Do not run multi-process tests with the unix-excl VFS. + # + if {$tn==1 && [permutation]=="unix-excl"} continue + faultsim_delete_and_reopen proc code1 {tcl} { uplevel #0 $tcl } diff --git a/test/permutations.test b/test/permutations.test index 04c0826eea..95dbb5cf35 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -617,6 +617,16 @@ test_suite "onefile" -description { rollback.test select1.test select2.test select3.test } +# Run some tests using the "unix-excl" VFS. +# +test_suite "unix-excl" -description { + Run some tests using the "unix-excl" VFS +} -initialize { + set ::G(perm:sqlite3_args) [list -vfs unix-excl] +} -files { + shmlock.test +} + # Run some tests using UTF-16 databases. # test_suite "utf16" -description { diff --git a/test/shmlock.test b/test/shmlock.test new file mode 100644 index 0000000000..6910758b82 --- /dev/null +++ b/test/shmlock.test @@ -0,0 +1,173 @@ +# 2018 December 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix shmlock + +ifcapable !wal {finish_test ; return } + +sqlite3 db2 test.db +sqlite3 db3 test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} {wal} +do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2} +do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2} + +foreach {tn dbhandle cmd res} { + 1 db {shared lock 7 1} OK + 2 db2 {exclusive lock 7 1} BUSY + 3 db {shared unlock 7 1} OK + 4 db2 {exclusive lock 7 1} OK + 5 db {shared lock 7 1} BUSY + 6 db {exclusive lock 7 1} BUSY + 7 db2 {exclusive unlock 7 1} OK + + 8 db {exclusive lock 0 8} OK + 9 db {exclusive unlock 0 8} OK + 10 db2 {exclusive lock 0 8} OK + 11 db2 {exclusive unlock 0 8} OK + + 12 db {shared lock 0 1} OK + 13 db2 {shared lock 0 1} OK + 14 db3 {shared lock 0 1} OK + 15 db3 {shared unlock 0 1} OK + 16 db3 {exclusive lock 0 1} BUSY + 17 db2 {shared unlock 0 1} OK + 18 db3 {exclusive lock 0 1} BUSY + 19 db {shared unlock 0 1} OK + 20 db3 {exclusive lock 0 1} OK + 21 db3 {exclusive unlock 0 1} OK + + 22 db {shared lock 3 1} OK + 23 db2 {exclusive lock 2 2} BUSY + 24 db {shared lock 2 1} OK + 25 db2 {exclusive lock 0 5} BUSY + 26 db2 {exclusive lock 0 4} BUSY + 27 db2 {exclusive lock 0 3} BUSY + 28 db {shared unlock 3 1} OK + 29 db2 {exclusive lock 2 2} BUSY + 28 db {shared unlock 2 1} OK + 29 db2 {exclusive lock 2 2} OK + 29 db2 {exclusive unlock 2 2} OK +} { + do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res" +} + +db close +db2 close +db3 close + +if {[permutation]=="unix-excl"} { + do_test 2.0 { + for {set i 0} {$i < 256} {incr i} { + sqlite3 db$i test.db + execsql { SELECT * FROM t1 } db$i + } + for {set i 0} {$i < 255} {incr i} { + set rc [vfs_shmlock db$i main shared lock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main shared lock 4 1 + } {SQLITE_BUSY} + + do_test 2.1 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + do_test 2.2 { vfs_shmlock db0 main shared unlock 4 1 } SQLITE_OK + do_test 2.3 { vfs_shmlock db255 main shared lock 4 1 } SQLITE_OK + do_test 2.4 { vfs_shmlock db255 main shared unlock 4 1 } SQLITE_OK + do_test 2.5 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + + do_test 2.6 { + for {set i 1} {$i < 255} {incr i} { + set rc [vfs_shmlock db255 main exclusive lock 4 1] + if {$rc != "SQLITE_BUSY"} { error $rc } + set rc [vfs_shmlock db$i main shared unlock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main exclusive lock 4 1 + } {SQLITE_OK} + + vfs_shmlock db255 main exclusive unlock 4 1 + + for {set i 0} {$i < 256} {incr i} { + db$i close + } +} + +sqlite3 db0 test.db +sqlite3 db1 test.db +do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2} +do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2} + +set L(0) {n n n n n n n n} +set L(1) {n n n n n n n n} +proc random_lock_test {idx} { + global L + set iSlot [expr int(rand()*8)] + if {[expr int(rand()*2)]} { + # Unlock operation + if {[lindex $L($idx) $iSlot]!="n"} { + vfs_shmlock db$idx main [lindex $L($idx) $iSlot] unlock $iSlot 1 + lset L($idx) $iSlot n + } + } else { + # Lock operation + if {[lindex $L($idx) $iSlot]=="n"} { + set locktype [lindex {e s} [expr int(rand()*2)]] + set n 1 + if {$locktype=="e"} { + for {set l $iSlot} {$l<8 && [lindex $L($idx) $l]=="n"} {incr l} {} + set n [expr int(rand()*($l-$iSlot))+1] + # puts "iSlot=$iSlot l=$l L=$L($idx)" + # puts "$iSlot $n" + } + set res [vfs_shmlock db$idx main $locktype lock $iSlot $n] + + set bBusy 0 + for {set i $iSlot} {$i<($iSlot+$n)} {incr i} { + set other [lindex $L([expr ($idx+1)%2]) $i] + if {($other!="n" && $locktype=="e")||($other=="e" && $locktype=="s")} { + if {$res != "SQLITE_BUSY"} { error "BUSY not detected" } + set bBusy 1 + break + } + } + + if {$bBusy==0} { + if {$res != "SQLITE_OK"} { error "BUSY false-positive" } + for {set i $iSlot} {$i<($iSlot+$n)} {incr i} { + lset L($idx) $i $locktype + } + } + } + } +} + +set nStep 100000 +for {set i 0} {$i < $nStep} {incr i} { + random_lock_test 0 + random_lock_test 1 +} + +db0 close +db1 close + +finish_test + + diff --git a/test/wal.test b/test/wal.test index bb164bb76a..a003b6ad20 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1297,51 +1297,53 @@ do_test wal-19.4 { # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # -do_test wal-20.1 { - catch {db close} - forcedelete test.db test.db-wal test.db-journal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(randomblob(900)); - SELECT count(*) FROM t1; - } -} {wal 1} -do_test wal-20.2 { - set ::buddy [launch_testfixture] - testfixture $::buddy { +if {[permutation]!="unix-excl"} { + do_test wal-20.1 { + catch {db close} + forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db - db transaction { db eval { - PRAGMA wal_autocheckpoint = 0; - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ - } } - } -} {0} -do_test wal-20.3 { - close $::buddy - execsql { PRAGMA wal_checkpoint } - execsql { SELECT count(*) FROM t1 } -} {16384} -do_test wal-20.4 { - db close - sqlite3 db test.db - execsql { SELECT count(*) FROM t1 } -} {16384} -integrity_check wal-20.5 + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(900)); + SELECT count(*) FROM t1; + } + } {wal 1} + do_test wal-20.2 { + set ::buddy [launch_testfixture] + testfixture $::buddy { + sqlite3 db test.db + db transaction { db eval { + PRAGMA wal_autocheckpoint = 0; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ + } } + } + } {0} + do_test wal-20.3 { + close $::buddy + execsql { PRAGMA wal_checkpoint } + execsql { SELECT count(*) FROM t1 } + } {16384} + do_test wal-20.4 { + db close + sqlite3 db test.db + execsql { SELECT count(*) FROM t1 } + } {16384} + integrity_check wal-20.5 +} catch { db2 close } catch { db close }