From e52854a9e6d3afacf60a690c1d31dc256da8cf1f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Nov 2023 19:22:50 +0000 Subject: [PATCH 1/3] In SQLITE_ENABLE_SETLK_TIMEOUT builds, use blocking locks in place of sleep() when opening a read-transaction. FossilOrigin-Name: a51ef39998e25e86bd0600e71d15011b12e05f4319608018293bdaecb09e8c97 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 13 +++++++++++++ src/wal.c | 53 ++++++++++++++++++++++++++++++++++----------------- 4 files changed, 59 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index bc7b1ca18c..2ccefe87c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--buildonly\sand\s--dryrun\soptions\sto\stestrunner.tcl. -D 2023-11-24T20:14:18.100 +C In\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds,\suse\sblocking\slocks\sin\splace\sof\ssleep()\swhen\sopening\sa\sread-transaction. +D 2023-11-27T19:22:50.016 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -712,7 +712,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c dc5404b56da7fb13cf272ddb94c3753cf9e82d32a65cba35dbb6aadcb849419c +F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 987ab3a2cd9065d62e9955474470ff733445e2357432a67e3d0f5a8f9313e334 @@ -810,7 +810,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 69aa3ce60b2862a24cd86bb528e653e2137388ead258ef64db49ec9038807f5f +F src/wal.c f64bf6799dbbd346a9e2aae9b96e51dbbb222d24a7e7509db85fb0144e8d9f38 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e @@ -2143,8 +2143,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8ea0b58b37f4052ded448e595d6d2992988a33b8ecfe58d68f20532f8cb5a37 -R a09b448e8fa45e3193b13a1fe5b499b2 +P a0c87ae9d3db914d18e2c8811db0d0ae3ad7b15c63de84fa975efce28bace27e +R 35e10752e16b85b0bde20be6e18f8d14 +T *branch * blocking-dms-lock +T *sym-blocking-dms-lock * +T -sym-trunk * U dan -Z ce0098ca9d7841ef7ea900cfd937f5a5 +Z fd8676cc431af47934a9227125e3da50 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5863b29096..9745d59d7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0c87ae9d3db914d18e2c8811db0d0ae3ad7b15c63de84fa975efce28bace27e \ No newline at end of file +a51ef39998e25e86bd0600e71d15011b12e05f4319608018293bdaecb09e8c97 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index dab03c97fb..7362a13206 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4595,7 +4595,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ pShmNode->isUnlocked = 1; rc = SQLITE_READONLY_CANTINIT; }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* Do not use a blocking lock here. If the lock cannot be obtained + ** immediately, it means some other connection is truncating the + ** *-shm file. And after it has done so, it will not release its + ** lock, but only downgrade it to a shared lock. So no point in + ** blocking here. The call below to obtain the shared DMS lock may + ** use a blocking lock. */ + int iSaveTimeout = pDbFd->iBusyTimeout; + pDbFd->iBusyTimeout = 0; +#endif rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pDbFd->iBusyTimeout = iSaveTimeout; +#endif /* The first connection to attach must truncate the -shm file. We ** truncate to 3 bytes (an arbitrary small number, less than the ** -shm header size) rather than 0 as a system debugging aid, to diff --git a/src/wal.c b/src/wal.c index d83f361d64..c610c14d02 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2004,6 +2004,19 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + + +/* +** Attempt to enable blocking locks that block for nMs ms. Return 1 if +** blocking locks are successfully enabled, or 0 otherwise. +*/ +static int walEnableBlockingMs(Wal *pWal, int nMs){ + int rc = sqlite3OsFileControl( + pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs + ); + return (rc==SQLITE_OK); +} + /* ** Attempt to enable blocking locks. Blocking locks are enabled only if (a) ** they are supported by the VFS, and (b) the database handle is configured @@ -2015,11 +2028,7 @@ static int walEnableBlocking(Wal *pWal){ if( pWal->db ){ int tmout = pWal->db->busyTimeout; if( tmout ){ - int rc; - rc = sqlite3OsFileControl( - pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout - ); - res = (rc==SQLITE_OK); + res = walEnableBlockingMs(pWal, tmout); } } return res; @@ -2068,20 +2077,10 @@ void sqlite3WalDb(Wal *pWal, sqlite3 *db){ pWal->db = db; } -/* -** Take an exclusive WRITE lock. Blocking if so configured. -*/ -static int walLockWriter(Wal *pWal){ - int rc; - walEnableBlocking(pWal); - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - walDisableBlocking(pWal); - return rc; -} #else # define walEnableBlocking(x) 0 # define walDisableBlocking(x) -# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1) +# define walEnableBlockingMs(pWal, ms) 0 # define sqlite3WalDb(pWal, db) #endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ @@ -2682,7 +2681,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ } }else{ int bWriteLock = pWal->writeLock; - if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){ + if( bWriteLock + || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) + ){ pWal->writeLock = 1; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); @@ -2690,7 +2691,8 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ /* If the wal-index header is still malformed even while holding ** a WRITE lock, it can only mean that the header is corrupted and ** needs to be reconstructed. So run recovery to do exactly that. - */ + ** Disable blocking locks first. */ + walDisableBlocking(pWal); rc = walIndexRecover(pWal); *pChanged = 1; } @@ -2987,6 +2989,17 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return SQLITE_PROTOCOL; } if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor + ** to block for locks for approximately nDelay us. This affects two + ** locks (a) the WRITER lock taken in walIndexReadHdr() if the first + ** attempted read fails and (b) the shared lock taken on the DMS slot + ** in os_unix.c. Both of these locks are attempted from within the + ** call to walIndexReadHdr() below. */ + if( cnt>=10 && !useWal && walEnableBlockingMs(pWal, (nDelay+999)/1000) ){ + nDelay = 1; + } +#endif sqlite3OsSleep(pWal->pVfs, nDelay); } @@ -2995,6 +3008,10 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( pWal->bShmUnreliable==0 ){ rc = walIndexReadHdr(pWal, pChanged); } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + walDisableBlocking(pWal); + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to From bae2d0f2ebfa7385890adae075e998e86252b59e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Nov 2023 20:37:03 +0000 Subject: [PATCH 2/3] Have SQLITE_ENABLE_SETLK_TIMEOUT builds block when locking a read-lock slot. FossilOrigin-Name: f797baf47cf7859cfd8ce248f4f3087af4551a7040af990333426e5a7c269504 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/wal.c | 8 +++++++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2ccefe87c7..82af9ff683 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds,\suse\sblocking\slocks\sin\splace\sof\ssleep()\swhen\sopening\sa\sread-transaction. -D 2023-11-27T19:22:50.016 +C Have\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds\sblock\swhen\slocking\sa\sread-lock\sslot. +D 2023-11-27T20:37:03.543 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -810,7 +810,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c f64bf6799dbbd346a9e2aae9b96e51dbbb222d24a7e7509db85fb0144e8d9f38 +F src/wal.c 86eaddc56b124de2df14dfaff292ca7bfb862dbd68728105547ddd6a1e799227 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e @@ -2143,11 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0c87ae9d3db914d18e2c8811db0d0ae3ad7b15c63de84fa975efce28bace27e -R 35e10752e16b85b0bde20be6e18f8d14 -T *branch * blocking-dms-lock -T *sym-blocking-dms-lock * -T -sym-trunk * +P a51ef39998e25e86bd0600e71d15011b12e05f4319608018293bdaecb09e8c97 +R 50c3db5d407d996f614a56ca5cb31c30 U dan -Z fd8676cc431af47934a9227125e3da50 +Z 3f1449733c562fe4e5bdc1215442e17e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9745d59d7d..8a036325f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a51ef39998e25e86bd0600e71d15011b12e05f4319608018293bdaecb09e8c97 \ No newline at end of file +f797baf47cf7859cfd8ce248f4f3087af4551a7040af990333426e5a7c269504 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index c610c14d02..a15114710b 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2959,6 +2959,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ u32 mxFrame; /* Wal frame to lock to */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int nBlockTmout = 0; +#endif assert( pWal->readLock<0 ); /* Not currently locked */ @@ -2996,7 +2999,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** attempted read fails and (b) the shared lock taken on the DMS slot ** in os_unix.c. Both of these locks are attempted from within the ** call to walIndexReadHdr() below. */ - if( cnt>=10 && !useWal && walEnableBlockingMs(pWal, (nDelay+999)/1000) ){ + nBlockTmout = (nDelay+998) / 1000; + if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){ nDelay = 1; } #endif @@ -3126,7 +3130,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } + (void)walEnableBlockingMs(pWal, nBlockTmout); rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + walDisableBlocking(pWal); if( rc ){ return rc==SQLITE_BUSY ? WAL_RETRY : rc; } From b8950e0f45656a96c09d5721e564fa0b8ef8e79a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Nov 2023 15:29:04 +0000 Subject: [PATCH 3/3] Handle an SQLITE_BUSY_TIMEOUT error if one occurs while attempting a shared lock on a read-lock slot. FossilOrigin-Name: 5fbf3906d272df3eb981f67455eb35f649ad2774cba9fc3f077b28d9bef3f0cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 82af9ff683..c28f30eb6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds\sblock\swhen\slocking\sa\sread-lock\sslot. -D 2023-11-27T20:37:03.543 +C Handle\san\sSQLITE_BUSY_TIMEOUT\serror\sif\sone\soccurs\swhile\sattempting\sa\sshared\slock\son\sa\sread-lock\sslot. +D 2023-11-28T15:29:04.012 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -810,7 +810,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 86eaddc56b124de2df14dfaff292ca7bfb862dbd68728105547ddd6a1e799227 +F src/wal.c fb52b05da32d500575a9b4ce0bb1c03b69876ea0f0873d4a8b0ca262da34e540 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a51ef39998e25e86bd0600e71d15011b12e05f4319608018293bdaecb09e8c97 -R 50c3db5d407d996f614a56ca5cb31c30 +P f797baf47cf7859cfd8ce248f4f3087af4551a7040af990333426e5a7c269504 +R c6e742d8a411bdf1bc8b658a51b789de U dan -Z 3f1449733c562fe4e5bdc1215442e17e +Z 58f292c9c356eaa52cbcdd405bfb61bb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8a036325f7..360d2aad65 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f797baf47cf7859cfd8ce248f4f3087af4551a7040af990333426e5a7c269504 \ No newline at end of file +5fbf3906d272df3eb981f67455eb35f649ad2774cba9fc3f077b28d9bef3f0cb \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index a15114710b..2bdaa1c3fb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3134,7 +3134,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); walDisableBlocking(pWal); if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; + assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); + return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; } /* Now that the read-lock has been obtained, check that neither the ** value in the aReadMark[] array or the contents of the wal-index