From 24fb9ffbc137b67829f240431b4ea58890bc437e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Dec 2018 18:58:58 +0000 Subject: [PATCH 1/5] Experiment with using an atomic CPU primitive instead of a mutex for intra-process locking with the unix-excl VFS. FossilOrigin-Name: 8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 --- manifest | 26 ++++++---- manifest.uuid | 2 +- src/os_unix.c | 85 +++++++++++++++++++++++++++++++ src/test_vfs.c | 54 ++++++++++++++++++++ test/lock_common.tcl | 10 ++-- test/permutations.test | 10 ++++ test/shmlock.test | 111 +++++++++++++++++++++++++++++++++++++++++ test/wal.test | 90 +++++++++++++++++---------------- 8 files changed, 328 insertions(+), 60 deletions(-) create mode 100644 test/shmlock.test diff --git a/manifest b/manifest index 720ed6ca6b..f8b2c4a247 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Invoking\sthe\ssqlite3_trace()\sor\ssqlite3_trace_v2()\sinterfaces\scancels\nany\ssqlite3_profile()\sthat\sis\srunning. -D 2018-12-06T03:59:25.918 +C Experiment\swith\susing\san\satomic\sCPU\sprimitive\sinstead\sof\sa\smutex\sfor\nintra-process\slocking\swith\sthe\sunix-excl\sVFS. +D 2018-12-06T18:58:58.578 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -491,7 +491,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c f6e91b8fd82af7afbfd073c4974ad6cdb8e62d9f65ceddb45167835a0567fdc0 +F src/os_unix.c dcf08ca4f7092507015391b078f9ab866e8fcc6fba22560086fe1eec031edd18 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 @@ -563,7 +563,7 @@ F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 F src/test_tclsh.c 06317648b0d85a85fd823f7973b55535c59a3156c1ef59394fe511f932cfa78d F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 -F src/test_vfs.c 112f1f9271c33c211812e0e681830a84262dac065da58579ff49f9cefec97d4f +F src/test_vfs.c 194b8e78e9b4279bc69693cb5cfc343e0a08482c430a571ec96d80440e337a44 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215 @@ -1078,7 +1078,7 @@ F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37 F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 -F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035 +F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083 F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -1181,7 +1181,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cf0b4e498db1d0143c19641d4420df7cc27fab2c95ed0abd2c7c5753beab25b8 +F test/permutations.test c2e5f618a71d1a7e50216325dd056f6ec16439b3c98d87bcd2df381297e5dc97 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1285,6 +1285,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013 +F test/shmlock.test 2d8d2401fb001414a817d61b2c4983215d546c71d48da517a0a5015422997628 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1592,7 +1593,7 @@ F test/vtabJ.test d7b73675708cf63cfcb9d443bb451fc01a028347275b7311e51f9fdf3ca675 F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad -F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 +F test/wal.test cdf0ca6cc0447520d19ef1c83287824ebeb3e82d75af856511ba96841a79fc9b F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1782,7 +1783,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de -R e33573679d2b83476b49b39d100df865 -U drh -Z 9b63335dca170db3a75ac781c84ad8b2 +P ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f +R 421ea33fec1ec5096b7fb80f863def19 +T *branch * mutexfree-shmlock +T *sym-mutexfree-shmlock * +T -sym-trunk * +U dan +Z 737072d87778273578811397782f507c diff --git a/manifest.uuid b/manifest.uuid index 787485caf5..f3e776b8e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f \ No newline at end of file +8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f20763e5b0..946c5d3d65 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,9 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Turn this feature on in all builds for now */ +#define SQLITE_MUTEXFREE_SHMLOCK 1 + /* ** There are various methods for file locking used for concurrency ** control: @@ -4239,8 +4242,38 @@ struct unixShmNode { u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif + +#ifdef SQLITE_MUTEXFREE_SHMLOCK + /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks + ** are stored in the following 64-bit value. There are in total 8 + ** shm-locking slots, each of which are assigned 8-bits from the 64-bit + ** value. The least-significant 8 bits correspond to shm-locking slot + ** 0, and so on. + ** + ** If the 8-bits corresponding to a shm-locking locking slot are set to + ** 0xFF, then a write-lock is held on the slot. Or, if they are set to + ** a non-zero value smaller than 0xFF, then they represent the total + ** number of read-locks held on the slot. There is no way to distinguish + ** between a write-lock and 255 read-locks. */ + u64 lockmask; +#endif }; +/* +** Atomic CAS primitive used in multi-process mode. Equivalent to: +** +** int unixCompareAndSwap(u32 *ptr, u32 oldval, u32 newval){ +** if( *ptr==oldval ){ +** *ptr = newval; +** return 1; +** } +** return 0; +** } +*/ +#define unixCompareAndSwap(ptr,oldval,newval) \ + __sync_bool_compare_and_swap(ptr,oldval,newval) + + /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. @@ -4800,6 +4833,58 @@ static int unixShmLock( mask = (1<<(ofst+n)) - (1<1 || mask==(1<pInode->bProcessLock ){ + + while( 1 ){ + u64 lockmask = pShmNode->lockmask; + u64 newmask = lockmask; + int i; + for(i=ofst; i> (ix8)) & 0xFF; + if( flags & SQLITE_SHM_UNLOCK ){ + if( flags & SQLITE_SHM_EXCLUSIVE ){ + if( p->exclMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<sharedMask & (1 << i) ){ + newmask = newmask & ~((u64)0xFF<exclMask & (1 << i))==0 ){ + newmask = newmask | ((u64)0xFF<sharedMask & (1 << i))==0 ){ + newmask = newmask & ~((u64)0xFF<lockmask, lockmask, newmask) ) break; + } + + if( flags & SQLITE_SHM_UNLOCK ){ + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + }else if( flags & SQLITE_SHM_EXCLUSIVE ){ + p->exclMask |= mask; + }else{ + p->sharedMask |= mask; + } + + return SQLITE_OK; + } +#endif + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ diff --git a/src/test_vfs.c b/src/test_vfs.c index 4a98ac214c..645b1447c2 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -1563,8 +1563,62 @@ static int SQLITE_TCLAPI testvfs_cmd( return TCL_ERROR; } +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern const char *sqlite3ErrName(int); + +/* +** tclcmd: vfs_shmlock DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N +*/ +static int SQLITE_TCLAPI test_vfs_shmlock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *azArg1[] = {"shared", "exclusive", 0}; + const char *azArg2[] = {"lock", "unlock", 0}; + sqlite3 *db = 0; + int rc = SQLITE_OK; + const char *zDbname = 0; + int iArg1 = 0; + int iArg2 = 0; + int iOffset = 0; + int n = 0; + sqlite3_file *pFd; + + if( objc!=7 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB DBNAME (shared|exclusive) (lock|unlock) OFFSET N" + ); + return TCL_ERROR; + } + + zDbname = Tcl_GetString(objv[2]); + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIndexFromObj(interp, objv[3], azArg1, "ARG", 0, &iArg1) + || Tcl_GetIndexFromObj(interp, objv[4], azArg2, "ARG", 0, &iArg2) + || Tcl_GetIntFromObj(interp, objv[5], &iOffset) + || Tcl_GetIntFromObj(interp, objv[6], &n) + ){ + return TCL_ERROR; + } + + sqlite3_file_control(db, zDbname, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + if( pFd==0 ){ + return TCL_ERROR; + } + rc = pFd->pMethods->xShmLock(pFd, iOffset, n, + (iArg1==0 ? SQLITE_SHM_SHARED : SQLITE_SHM_EXCLUSIVE) + | (iArg2==0 ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_OK; +} + + int Sqlitetestvfs_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "testvfs", testvfs_cmd, 0, 0); + Tcl_CreateObjCommand(interp, "vfs_shmlock", test_vfs_shmlock, 0, 0); return TCL_OK; } diff --git a/test/lock_common.tcl b/test/lock_common.tcl index a758e7af2e..3e1821bab0 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -15,18 +15,20 @@ proc do_multiclient_test {varname script} { - foreach code [list { + foreach {tn code} [list 1 { if {[info exists ::G(valgrind)]} { db close ; continue } set ::code2_chan [launch_testfixture] set ::code3_chan [launch_testfixture] proc code2 {tcl} { testfixture $::code2_chan $tcl } proc code3 {tcl} { testfixture $::code3_chan $tcl } - set tn 1 - } { + } 2 { proc code2 {tcl} { uplevel #0 $tcl } proc code3 {tcl} { uplevel #0 $tcl } - set tn 2 }] { + # Do not run multi-process tests with the unix-excl VFS. + # + if {$tn==1 && [permutation]=="unix-excl"} continue + faultsim_delete_and_reopen proc code1 {tcl} { uplevel #0 $tcl } diff --git a/test/permutations.test b/test/permutations.test index 6aa812e8af..a4c21e67d8 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -607,6 +607,16 @@ test_suite "onefile" -description { rollback.test select1.test select2.test select3.test } +# Run some tests using the "unix-excl" VFS. +# +test_suite "unix-excl" -description { + Run some tests using the "unix-excl" VFS +} -initialize { + set ::G(perm:sqlite3_args) [list -vfs unix-excl] +} -files { + shmlock.test +} + # Run some tests using UTF-16 databases. # test_suite "utf16" -description { diff --git a/test/shmlock.test b/test/shmlock.test new file mode 100644 index 0000000000..ac79ac44d3 --- /dev/null +++ b/test/shmlock.test @@ -0,0 +1,111 @@ +# 2018 December 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix shmlock + +ifcapable !wal {finish_test ; return } + +sqlite3 db2 test.db +sqlite3 db3 test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} {wal} +do_test 1.1 { execsql { SELECT * FROM t1 } db2 } {1 2} +do_test 1.2 { execsql { SELECT * FROM t1 } db3 } {1 2} + +foreach {tn dbhandle cmd res} { + 1 db {shared lock 7 1} OK + 2 db2 {exclusive lock 7 1} BUSY + 3 db {shared unlock 7 1} OK + 4 db2 {exclusive lock 7 1} OK + 5 db {shared lock 7 1} BUSY + 6 db {exclusive lock 7 1} BUSY + 7 db2 {exclusive unlock 7 1} OK + + 8 db {exclusive lock 0 8} OK + 9 db {exclusive unlock 0 8} OK + 10 db2 {exclusive lock 0 8} OK + 11 db2 {exclusive unlock 0 8} OK + + 12 db {shared lock 0 1} OK + 13 db2 {shared lock 0 1} OK + 14 db3 {shared lock 0 1} OK + 15 db3 {shared unlock 0 1} OK + 16 db3 {exclusive lock 0 1} BUSY + 17 db2 {shared unlock 0 1} OK + 18 db3 {exclusive lock 0 1} BUSY + 19 db {shared unlock 0 1} OK + 20 db3 {exclusive lock 0 1} OK + 21 db3 {exclusive unlock 0 1} OK + + 22 db {shared lock 3 1} OK + 23 db2 {exclusive lock 2 2} BUSY + 24 db {shared lock 2 1} OK + 25 db2 {exclusive lock 0 5} BUSY + 26 db2 {exclusive lock 0 4} BUSY + 27 db2 {exclusive lock 0 3} BUSY + 28 db {shared unlock 3 1} OK + 29 db2 {exclusive lock 2 2} BUSY + 28 db {shared unlock 2 1} OK + 29 db2 {exclusive lock 2 2} OK + 29 db2 {exclusive unlock 2 2} OK +} { + do_test 1.3.$tn [list vfs_shmlock $dbhandle main {*}$cmd] "SQLITE_$res" +} + +db close +db2 close +db3 close + +if {[permutation]=="unix-excl"} { + do_test 2.0 { + for {set i 0} {$i < 256} {incr i} { + sqlite3 db$i test.db + execsql { SELECT * FROM t1 } db$i + } + for {set i 0} {$i < 255} {incr i} { + set rc [vfs_shmlock db$i main shared lock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main shared lock 4 1 + } {SQLITE_BUSY} + + do_test 2.1 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + do_test 2.2 { vfs_shmlock db0 main shared unlock 4 1 } SQLITE_OK + do_test 2.3 { vfs_shmlock db255 main shared lock 4 1 } SQLITE_OK + do_test 2.4 { vfs_shmlock db255 main shared unlock 4 1 } SQLITE_OK + do_test 2.5 { vfs_shmlock db255 main exclusive lock 4 1 } SQLITE_BUSY + + do_test 2.6 { + for {set i 1} {$i < 255} {incr i} { + set rc [vfs_shmlock db255 main exclusive lock 4 1] + if {$rc != "SQLITE_BUSY"} { error $rc } + set rc [vfs_shmlock db$i main shared unlock 4 1] + if {$rc != "SQLITE_OK"} { error $rc } + } + + vfs_shmlock db255 main exclusive lock 4 1 + } {SQLITE_OK} + + vfs_shmlock db255 main exclusive unlock 4 1 +} + +finish_test + + diff --git a/test/wal.test b/test/wal.test index bb164bb76a..a003b6ad20 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1297,51 +1297,53 @@ do_test wal-19.4 { # At one point, SQLite was failing to grow the mapping of the wal-index # file in step 3 and the checkpoint was corrupting the database file. # -do_test wal-20.1 { - catch {db close} - forcedelete test.db test.db-wal test.db-journal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(randomblob(900)); - SELECT count(*) FROM t1; - } -} {wal 1} -do_test wal-20.2 { - set ::buddy [launch_testfixture] - testfixture $::buddy { +if {[permutation]!="unix-excl"} { + do_test wal-20.1 { + catch {db close} + forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db - db transaction { db eval { - PRAGMA wal_autocheckpoint = 0; - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ - } } - } -} {0} -do_test wal-20.3 { - close $::buddy - execsql { PRAGMA wal_checkpoint } - execsql { SELECT count(*) FROM t1 } -} {16384} -do_test wal-20.4 { - db close - sqlite3 db test.db - execsql { SELECT count(*) FROM t1 } -} {16384} -integrity_check wal-20.5 + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(900)); + SELECT count(*) FROM t1; + } + } {wal 1} + do_test wal-20.2 { + set ::buddy [launch_testfixture] + testfixture $::buddy { + sqlite3 db test.db + db transaction { db eval { + PRAGMA wal_autocheckpoint = 0; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ + } } + } + } {0} + do_test wal-20.3 { + close $::buddy + execsql { PRAGMA wal_checkpoint } + execsql { SELECT count(*) FROM t1 } + } {16384} + do_test wal-20.4 { + db close + sqlite3 db test.db + execsql { SELECT count(*) FROM t1 } + } {16384} + integrity_check wal-20.5 +} catch { db2 close } catch { db close } From 5b5cda202718fcb178b66fb00c8a98bb0ea35a26 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Dec 2018 09:45:38 +0000 Subject: [PATCH 2/5] Avoid a mutex in-and-out in unixShmBarrier() on this branch. Use __sync_synchronize() instead. FossilOrigin-Name: a8c5fd86ce1d9b5815f82106f2e44797b743bbb3c44aee5f835ce8d278caf8f7 --- manifest | 16 +++++++--------- manifest.uuid | 2 +- src/os_unix.c | 4 ++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f8b2c4a247..7737a87ca3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiment\swith\susing\san\satomic\sCPU\sprimitive\sinstead\sof\sa\smutex\sfor\nintra-process\slocking\swith\sthe\sunix-excl\sVFS. -D 2018-12-06T18:58:58.578 +C Avoid\sa\smutex\sin-and-out\sin\sunixShmBarrier()\son\sthis\sbranch.\sUse\n__sync_synchronize()\sinstead. +D 2018-12-10T09:45:38.153 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -491,7 +491,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c dcf08ca4f7092507015391b078f9ab866e8fcc6fba22560086fe1eec031edd18 +F src/os_unix.c 8526ea4590b158dfa1b62831da9b50337d131f375aaa27c61567b345e9760cd2 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 @@ -1783,10 +1783,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f -R 421ea33fec1ec5096b7fb80f863def19 -T *branch * mutexfree-shmlock -T *sym-mutexfree-shmlock * -T -sym-trunk * +P 8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 +Q +280d1a72dc269b4402853159949d7a905d414786074f5d80e951c9341ebbd1b5 +R 124b36ab6f5a94c22a4c2edeb34525f1 U dan -Z 737072d87778273578811397782f507c +Z 3d15546b7a977bfe213c636690aee9c2 diff --git a/manifest.uuid b/manifest.uuid index f3e776b8e7..904b45f5cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 \ No newline at end of file +a8c5fd86ce1d9b5815f82106f2e44797b743bbb3c44aee5f835ce8d278caf8f7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 946c5d3d65..0c2cb98bdf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4974,12 +4974,16 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); +#ifdef SQLITE_MUTEXFREE_SHMLOCK + __sync_synchronize(); +#else sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ assert( fd->pMethods->xLock==nolockLock || unixFileMutexNotheld((unixFile*)fd) ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); +#endif } /* From c967a11569e50535326f7dcc97883a93d8962a9d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Dec 2018 15:24:29 +0000 Subject: [PATCH 3/5] Rework the SQLITE_MUTEXFREE_SHMLOCK code to reduce contention. FossilOrigin-Name: d9157dd176a2d18c6e02a2a0c7e16cef2da43bf44be9765e0363f34aebad23e9 --- manifest | 17 +++-- manifest.uuid | 2 +- src/os_unix.c | 147 ++++++++++++++++++++++++++++--------------- src/test_superlock.c | 21 +++++-- src/wal.c | 19 ++---- 5 files changed, 128 insertions(+), 78 deletions(-) diff --git a/manifest b/manifest index 7737a87ca3..30cc81f0ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\smutex\sin-and-out\sin\sunixShmBarrier()\son\sthis\sbranch.\sUse\n__sync_synchronize()\sinstead. -D 2018-12-10T09:45:38.153 +C Rework\sthe\sSQLITE_MUTEXFREE_SHMLOCK\scode\sto\sreduce\scontention. +D 2018-12-10T15:24:29.922 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -491,7 +491,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 8526ea4590b158dfa1b62831da9b50337d131f375aaa27c61567b345e9760cd2 +F src/os_unix.c 4b21c5148ca49fa136a6f404121080deee3e14f14ce07ea3cd132e15d40e5a93 F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 @@ -558,7 +558,7 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef -F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e +F src/test_superlock.c f4d4cc7319a608a54b7608158e8c7135fac19b88d6179e5bf17080e89d1f0278 F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 F src/test_tclsh.c 06317648b0d85a85fd823f7973b55535c59a3156c1ef59394fe511f932cfa78d F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc @@ -589,7 +589,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 3f4f653daf234fe713edbcbca3fec2350417d159d28801feabc702a22c4e213f +F src/wal.c d1c4f4edd172d4b9320d785304bad3efc433baf672621653017434ab87f3f336 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1783,8 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f4cb9dd3396bcaaf85dcdb4e3ae3c96f28a4d71d72665c4abf7c221370be626 -Q +280d1a72dc269b4402853159949d7a905d414786074f5d80e951c9341ebbd1b5 -R 124b36ab6f5a94c22a4c2edeb34525f1 +P a8c5fd86ce1d9b5815f82106f2e44797b743bbb3c44aee5f835ce8d278caf8f7 +R 2c73dae6f22740de28ff76884e3353b6 U dan -Z 3d15546b7a977bfe213c636690aee9c2 +Z 57d8b49adf3b9b02554cff5dcd73ea2d diff --git a/manifest.uuid b/manifest.uuid index 904b45f5cd..8342dba98b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8c5fd86ce1d9b5815f82106f2e44797b743bbb3c44aee5f835ce8d278caf8f7 \ No newline at end of file +d9157dd176a2d18c6e02a2a0c7e16cef2da43bf44be9765e0363f34aebad23e9 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 0c2cb98bdf..7c31d72511 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -48,6 +48,8 @@ /* Turn this feature on in all builds for now */ #define SQLITE_MUTEXFREE_SHMLOCK 1 +#define SQLITE_MFS_NSHARD 5 +#define SQLITE_MFS_EXCLUSIVE 255 /* ** There are various methods for file locking used for concurrency @@ -4255,7 +4257,10 @@ struct unixShmNode { ** a non-zero value smaller than 0xFF, then they represent the total ** number of read-locks held on the slot. There is no way to distinguish ** between a write-lock and 255 read-locks. */ - u64 lockmask; + struct LockingSlot { + u32 nLock; + u64 aPadding[7]; + } aMFSlot[3 + SQLITE_MFS_NSHARD*5]; #endif }; @@ -4294,6 +4299,9 @@ struct unixShm { u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_MUTEXFREE_SHMLOCK + u8 aMFCurrent[8]; /* Current slot used for each shared lock */ +#endif }; /* @@ -4798,6 +4806,83 @@ shmpage_out: return rc; } +#ifdef SQLITE_MUTEXFREE_SHMLOCK +static int unixMutexFreeShmlock( + unixFile *pFd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + struct LockMapEntry { + int iFirst; + int nSlot; + } aMap[9] = { + { 0, 1 }, + { 1, 1 }, + { 2, 1 }, + { 3+0*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+1*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+2*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+3*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+4*SQLITE_MFS_NSHARD, SQLITE_MFS_NSHARD }, + { 3+5*SQLITE_MFS_NSHARD, 0 }, + }; + + unixShm *p = pFd->pShm; /* The shared memory being locked */ + unixShm *pX; /* For looping over all siblings */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; + int iIncr; + u16 mask; /* Mask of locks to take or release */ + + if( flags & SQLITE_SHM_SHARED ){ + /* SHARED locks */ + u32 iOld, iNew, *ptr; + int iIncr = -1; + if( (flags & SQLITE_SHM_UNLOCK)==0 ){ + p->aMFCurrent[ofst] = (p->aMFCurrent[ofst] + 1) % aMap[ofst].nSlot; + iIncr = 1; + } + ptr = &pShmNode->aMFSlot[aMap[ofst].iFirst + p->aMFCurrent[ofst]].nLock; + do { + iOld = *ptr; + iNew = iOld + iIncr; + if( iNew>SQLITE_MFS_EXCLUSIVE ){ + return SQLITE_BUSY; + } + }while( 0==unixCompareAndSwap(ptr, iOld, iNew) ); + }else{ + /* EXCLUSIVE locks */ + int iFirst = aMap[ofst].iFirst; + int iLast = aMap[ofst+n].iFirst; + int i; + for(i=iFirst; iaMFSlot[i].nLock; + if( flags & SQLITE_SHM_UNLOCK ){ + assert( (*ptr)==SQLITE_MFS_EXCLUSIVE ); + *ptr = 0; + }else{ + u32 iOld; + do { + iOld = *ptr; + if( iOld>0 ){ + while( i>iFirst ){ + i--; + pShmNode->aMFSlot[i].nLock = 0; + } + return SQLITE_BUSY; + } + }while( 0==unixCompareAndSwap(ptr, iOld, SQLITE_MFS_EXCLUSIVE) ); + } + } + } + + return SQLITE_OK; +} +#else +# define unixMutexFreeShmlock(a,b,c,d) SQLITE_OK +#endif + /* ** Change the lock state for a shared-memory segment. ** @@ -4831,59 +4916,19 @@ static int unixShmLock( assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + if( pDbFd->pInode->bProcessLock ){ + return unixMutexFreeShmlock(pDbFd, ofst, n, flags); + } + mask = (1<<(ofst+n)) - (1<1 || mask==(1<pInode->bProcessLock ){ - - while( 1 ){ - u64 lockmask = pShmNode->lockmask; - u64 newmask = lockmask; - int i; - for(i=ofst; i> (ix8)) & 0xFF; - if( flags & SQLITE_SHM_UNLOCK ){ - if( flags & SQLITE_SHM_EXCLUSIVE ){ - if( p->exclMask & (1 << i) ){ - newmask = newmask & ~((u64)0xFF<sharedMask & (1 << i) ){ - newmask = newmask & ~((u64)0xFF<exclMask & (1 << i))==0 ){ - newmask = newmask | ((u64)0xFF<sharedMask & (1 << i))==0 ){ - newmask = newmask & ~((u64)0xFF<lockmask, lockmask, newmask) ) break; - } - - if( flags & SQLITE_SHM_UNLOCK ){ - p->sharedMask &= ~mask; - p->exclMask &= ~mask; - }else if( flags & SQLITE_SHM_EXCLUSIVE ){ - p->exclMask |= mask; - }else{ - p->sharedMask |= mask; - } - - return SQLITE_OK; + if( flags & SQLITE_SHM_LOCK ){ + assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==0 ); + assert( !(flags&SQLITE_SHM_EXCLUSIVE) || !(p->exclMask&mask) ); + }else{ + assert( !(flags&SQLITE_SHM_SHARED) || (p->sharedMask&mask)==mask ); + assert( !(flags&SQLITE_SHM_EXCLUSIVE) || (p->exclMask&mask)==mask ); } -#endif sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ diff --git a/src/test_superlock.c b/src/test_superlock.c index 45d0d623a0..d677ce5378 100644 --- a/src/test_superlock.c +++ b/src/test_superlock.c @@ -41,6 +41,8 @@ typedef struct SuperlockBusy SuperlockBusy; struct Superlock { sqlite3 *db; /* Database handle used to lock db */ int bWal; /* True if db is a WAL database */ + int bRecoveryLocked; /* True if WAL RECOVERY lock is held */ + int bReaderLocked; /* True if WAL READER locks are held */ }; typedef struct Superlock Superlock; @@ -107,12 +109,13 @@ static int superlockShmLock( ** Invoke the supplied busy-handler as required. */ static int superlockWalLock( - sqlite3 *db, /* Database handle open on WAL database */ + Superlock *pLock, /* Superlock handle */ SuperlockBusy *pBusy /* Busy handler wrapper object */ ){ int rc; /* Return code */ sqlite3_file *fd = 0; /* Main database file handle */ void volatile *p = 0; /* Pointer to first page of shared memory */ + sqlite3 *db = pLock->db; /* Obtain a pointer to the sqlite3_file object open on the main db file. */ rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd); @@ -121,8 +124,10 @@ static int superlockWalLock( /* Obtain the "recovery" lock. Normally, this lock is only obtained by ** clients running database recovery. */ + assert( pLock->bRecoveryLocked==0 ); rc = superlockShmLock(fd, 2, 1, pBusy); if( rc!=SQLITE_OK ) return rc; + pLock->bRecoveryLocked = 1; /* Zero the start of the first shared-memory page. This means that any ** clients that open read or write transactions from this point on will @@ -139,7 +144,9 @@ static int superlockWalLock( ** are held, it is guaranteed that there are no active reader, writer or ** checkpointer clients. */ + assert( pLock->bReaderLocked==0 ); rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy); + if( rc==SQLITE_OK ) pLock->bReaderLocked = 1; return rc; } @@ -156,8 +163,14 @@ void sqlite3demo_superunlock(void *pLock){ sqlite3_file *fd = 0; rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd); if( rc==SQLITE_OK ){ - fd->pMethods->xShmLock(fd, 2, 1, flags); - fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags); + if( p->bRecoveryLocked ){ + fd->pMethods->xShmLock(fd, 2, 1, flags); + p->bRecoveryLocked = 0; + } + if( p->bReaderLocked ){ + fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags); + p->bReaderLocked = 0; + } } } sqlite3_close(p->db); @@ -232,7 +245,7 @@ int sqlite3demo_superlock( if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){ rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = superlockWalLock(pLock->db, &busy); + rc = superlockWalLock(pLock, &busy); } } } diff --git a/src/wal.c b/src/wal.c index 4088bf2ec8..2883bd2d16 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2777,19 +2777,14 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); - /* It is possible that there is a checkpointer thread running - ** concurrent with this code. If this is the case, it may be that the - ** checkpointer has already determined that it will checkpoint + /* If it were possible for a checkpointer thread to run concurrent + ** with this code, it would be a problem. In this case, it could be + ** that the checkpointer has already determined that it will checkpoint ** snapshot X, where X is later in the wal file than pSnapshot, but ** has not yet set the pInfo->nBackfillAttempted variable to indicate - ** its intent. To avoid the race condition this leads to, ensure that - ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. - ** - ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing - ** this already? - */ - rc = walLockShared(pWal, WAL_CKPT_LOCK); + ** its intent. Fortunately this is not possible, as the call to + ** sqlite3WalSnapshotOpen() that sets pWal->pSnapshot also takes a + ** SHARED lock on the checkpointer slot. */ if( rc==SQLITE_OK ){ /* Check that the wal file has not been wrapped. Assuming that it has @@ -2808,8 +2803,6 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ rc = SQLITE_ERROR_SNAPSHOT; } - /* Release the shared CKPT lock obtained above. */ - walUnlockShared(pWal, WAL_CKPT_LOCK); pWal->minFrame = 1; } From 1d5430f85ed3c6181f792a392afce83fb4c89eec Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Dec 2018 15:51:05 +0000 Subject: [PATCH 4/5] Add extra tests to shmlock.test (direct testing of xShmLock methods). FossilOrigin-Name: d2c785f94cc6b704ce29147399e2c473fd0f4ff377f1379bd29178bf6d127c1a --- manifest | 12 ++++----- manifest.uuid | 2 +- test/shmlock.test | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 30cc81f0ed..16143f8169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sSQLITE_MUTEXFREE_SHMLOCK\scode\sto\sreduce\scontention. -D 2018-12-10T15:24:29.922 +C Add\sextra\stests\sto\sshmlock.test\s(direct\stesting\sof\sxShmLock\smethods). +D 2018-12-10T15:51:05.281 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -1285,7 +1285,7 @@ F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013 -F test/shmlock.test 2d8d2401fb001414a817d61b2c4983215d546c71d48da517a0a5015422997628 +F test/shmlock.test 3d1868f0386923c0592a235f2dd87ae52286b217bc695fbfd9d39a828e7be374 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8c5fd86ce1d9b5815f82106f2e44797b743bbb3c44aee5f835ce8d278caf8f7 -R 2c73dae6f22740de28ff76884e3353b6 +P d9157dd176a2d18c6e02a2a0c7e16cef2da43bf44be9765e0363f34aebad23e9 +R 24f7a75900edcf030e0982b0e1c98eeb U dan -Z 57d8b49adf3b9b02554cff5dcd73ea2d +Z d8fb14bd354e885b5baff90283e91e8d diff --git a/manifest.uuid b/manifest.uuid index 8342dba98b..abbf850327 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9157dd176a2d18c6e02a2a0c7e16cef2da43bf44be9765e0363f34aebad23e9 \ No newline at end of file +d2c785f94cc6b704ce29147399e2c473fd0f4ff377f1379bd29178bf6d127c1a \ No newline at end of file diff --git a/test/shmlock.test b/test/shmlock.test index ac79ac44d3..6910758b82 100644 --- a/test/shmlock.test +++ b/test/shmlock.test @@ -104,8 +104,70 @@ if {[permutation]=="unix-excl"} { } {SQLITE_OK} vfs_shmlock db255 main exclusive unlock 4 1 + + for {set i 0} {$i < 256} {incr i} { + db$i close + } } +sqlite3 db0 test.db +sqlite3 db1 test.db +do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2} +do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2} + +set L(0) {n n n n n n n n} +set L(1) {n n n n n n n n} +proc random_lock_test {idx} { + global L + set iSlot [expr int(rand()*8)] + if {[expr int(rand()*2)]} { + # Unlock operation + if {[lindex $L($idx) $iSlot]!="n"} { + vfs_shmlock db$idx main [lindex $L($idx) $iSlot] unlock $iSlot 1 + lset L($idx) $iSlot n + } + } else { + # Lock operation + if {[lindex $L($idx) $iSlot]=="n"} { + set locktype [lindex {e s} [expr int(rand()*2)]] + set n 1 + if {$locktype=="e"} { + for {set l $iSlot} {$l<8 && [lindex $L($idx) $l]=="n"} {incr l} {} + set n [expr int(rand()*($l-$iSlot))+1] + # puts "iSlot=$iSlot l=$l L=$L($idx)" + # puts "$iSlot $n" + } + set res [vfs_shmlock db$idx main $locktype lock $iSlot $n] + + set bBusy 0 + for {set i $iSlot} {$i<($iSlot+$n)} {incr i} { + set other [lindex $L([expr ($idx+1)%2]) $i] + if {($other!="n" && $locktype=="e")||($other=="e" && $locktype=="s")} { + if {$res != "SQLITE_BUSY"} { error "BUSY not detected" } + set bBusy 1 + break + } + } + + if {$bBusy==0} { + if {$res != "SQLITE_OK"} { error "BUSY false-positive" } + for {set i $iSlot} {$i<($iSlot+$n)} {incr i} { + lset L($idx) $i $locktype + } + } + } + } +} + +set nStep 100000 +for {set i 0} {$i < $nStep} {incr i} { + random_lock_test 0 + random_lock_test 1 +} + +db0 close +db1 close + finish_test From 107c5de8b2f976b08ed3b722c2136a25f4c35b2f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Dec 2018 16:52:31 +0000 Subject: [PATCH 5/5] Make SQLITE_MFS_NSHARD a compile time setting. FossilOrigin-Name: b9a741519055a8560ec1d0ab7d0c15ff8db53e740011825506cd2ede11c956f0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 16143f8169..2f7fef6936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\sshmlock.test\s(direct\stesting\sof\sxShmLock\smethods). -D 2018-12-10T15:51:05.281 +C Make\sSQLITE_MFS_NSHARD\sa\scompile\stime\ssetting. +D 2018-12-10T16:52:31.796 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -491,7 +491,7 @@ F src/os.c 8aeb0b0f40f8f5b0da03fe49706695adaf42d2f516ab95abc72e86c245e119de F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 4b21c5148ca49fa136a6f404121080deee3e14f14ce07ea3cd132e15d40e5a93 +F src/os_unix.c e191348d66fcbcf137cb2497de2c1e2ccca3caac0515a42c1a1d38043f519e9e F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9157dd176a2d18c6e02a2a0c7e16cef2da43bf44be9765e0363f34aebad23e9 -R 24f7a75900edcf030e0982b0e1c98eeb +P d2c785f94cc6b704ce29147399e2c473fd0f4ff377f1379bd29178bf6d127c1a +R 405c2e0a847d1dc9742f041e349c6a79 U dan -Z d8fb14bd354e885b5baff90283e91e8d +Z 0d67b443d35d0a267898c50a3a55aea3 diff --git a/manifest.uuid b/manifest.uuid index abbf850327..890ad2c80f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2c785f94cc6b704ce29147399e2c473fd0f4ff377f1379bd29178bf6d127c1a \ No newline at end of file +b9a741519055a8560ec1d0ab7d0c15ff8db53e740011825506cd2ede11c956f0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7c31d72511..435e4cad08 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -48,8 +48,13 @@ /* Turn this feature on in all builds for now */ #define SQLITE_MUTEXFREE_SHMLOCK 1 -#define SQLITE_MFS_NSHARD 5 #define SQLITE_MFS_EXCLUSIVE 255 +#ifndef SQLITE_MFS_NSHARD +# define SQLITE_MFS_NSHARD 8 +#endif +#if SQLITE_MFS_NSHARD<1 +# error "SQLITE_MFS_NSHARD must be greater than 0" +#endif /* ** There are various methods for file locking used for concurrency