1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-27 20:41:58 +03:00

Add test app for experimenting with multi-worker OPFS concurrency. Tweak OPFS VFS to significantly improve the otherwise "unfortunate" concurrency situation.

FossilOrigin-Name: 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c
This commit is contained in:
stephan
2022-11-21 03:50:52 +00:00
parent ae276719f0
commit 27c4cd183d
11 changed files with 364 additions and 96 deletions

View File

@ -53,7 +53,7 @@ const state = Object.create(null);
2 = warnings and errors
3 = debug, warnings, and errors
*/
state.verbose = 2;
state.verbose = 1;
const loggers = {
0:console.error.bind(console),
@ -150,70 +150,6 @@ const getDirForFilename = async function f(absFilename, createDirs = false){
return [dh, filename];
};
/**
An error class specifically for use with getSyncHandle(), the goal
of which is to eventually be able to distinguish unambiguously
between locking-related failures and other types, noting that we
cannot currently do so because createSyncAccessHandle() does not
define its exceptions in the required level of detail.
*/
class GetSyncHandleError extends Error {
constructor(errorObject, ...msg){
super();
this.error = errorObject;
this.message = [
...msg, ': Original exception ['+errorObject.name+']:',
errorObject.message
].join(' ');
this.name = 'GetSyncHandleError';
}
};
/**
Returns the sync access handle associated with the given file
handle object (which must be a valid handle object, as created by
xOpen()), lazily opening it if needed.
In order to help alleviate cross-tab contention for a dabase,
if an exception is thrown while acquiring the handle, this routine
will wait briefly and try again, up to 3 times. If acquisition
still fails at that point it will give up and propagate the
exception.
*/
const getSyncHandle = async (fh)=>{
if(!fh.syncHandle){
const t = performance.now();
log("Acquiring sync handle for",fh.filenameAbs);
const maxTries = 4, msBase = 300;
let i = 1, ms = msBase;
for(; true; ms = msBase * ++i){
try {
//if(i<3) toss("Just testing getSyncHandle() wait-and-retry.");
//TODO? A config option which tells it to throw here
//randomly every now and then, for testing purposes.
fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
break;
}catch(e){
if(i === maxTries){
throw new GetSyncHandleError(
e, "Error getting sync handle.",maxTries,
"attempts failed.",fh.filenameAbs
);
}
warn("Error getting sync handle. Waiting",ms,
"ms and trying again.",fh.filenameAbs,e);
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
}
}
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
if(!fh.xLock){
__autoLocks.add(fh.fid);
log("Auto-locked",fh.fid,fh.filenameAbs);
}
}
return fh.syncHandle;
};
/**
If the given file-holding object has a sync handle attached to it,
that handle is remove and asynchronously closed. Though it may
@ -253,6 +189,101 @@ const closeSyncHandleNoThrow = async (fh)=>{
}
};
/* Release all auto-locks. */
const closeAutoLocks = async ()=>{
if(__autoLocks.size){
/* Release all auto-locks. */
for(const fid of __autoLocks){
const fh = __openFiles[fid];
await closeSyncHandleNoThrow(fh);
log("Auto-unlocked",fid,fh.filenameAbs);
}
}
};
/**
An error class specifically for use with getSyncHandle(), the goal
of which is to eventually be able to distinguish unambiguously
between locking-related failures and other types, noting that we
cannot currently do so because createSyncAccessHandle() does not
define its exceptions in the required level of detail.
*/
class GetSyncHandleError extends Error {
constructor(errorObject, ...msg){
super();
this.error = errorObject;
this.message = [
...msg, ': Original exception ['+errorObject.name+']:',
errorObject.message
].join(' ');
this.name = 'GetSyncHandleError';
}
};
GetSyncHandleError.convertRc = (e,rc)=>{
if(1){
/* This approach returns SQLITE_LOCKED to the C API
when getSyncHandle() fails but makes the very
wild assumption that such a failure _is_ a locking
error. In practice that appears to be the most
common error, by far, but we cannot unambiguously
distinguish that from other errors.
This approach demonstrably reduces concurrency-related
errors but is highly questionable.
*/
return (e instanceof GetSyncHandleError)
? state.sq3Codes.SQLITE_LOCKED
: rc;
}else{
return ec;
}
}
/**
Returns the sync access handle associated with the given file
handle object (which must be a valid handle object, as created by
xOpen()), lazily opening it if needed.
In order to help alleviate cross-tab contention for a dabase,
if an exception is thrown while acquiring the handle, this routine
will wait briefly and try again, up to 3 times. If acquisition
still fails at that point it will give up and propagate the
exception.
*/
const getSyncHandle = async (fh)=>{
if(!fh.syncHandle){
const t = performance.now();
log("Acquiring sync handle for",fh.filenameAbs);
const maxTries = 4, msBase = 300;
let i = 1, ms = msBase;
for(; true; ms = msBase * ++i){
try {
//if(i<3) toss("Just testing getSyncHandle() wait-and-retry.");
//TODO? A config option which tells it to throw here
//randomly every now and then, for testing purposes.
fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
break;
}catch(e){
if(i === maxTries){
throw new GetSyncHandleError(
e, "Error getting sync handle.",maxTries,
"attempts failed.",fh.filenameAbs
);
}
warn("Error getting sync handle. Waiting",ms,
"ms and trying again.",fh.filenameAbs,e);
//await closeAutoLocks();
Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
}
}
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
if(!fh.xLock){
__autoLocks.add(fh.fid);
log("Auto-locked",fh.fid,fh.filenameAbs);
}
}
return fh.syncHandle;
};
/**
Stores the given value at state.sabOPView[state.opIds.rc] and then
Atomics.notify()'s it.
@ -451,7 +482,7 @@ const vfsAsyncImpls = {
rc = 0;
}catch(e){
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR;
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR);
}
wTimeEnd();
storeAndNotify('xFileSize', rc);
@ -471,7 +502,7 @@ const vfsAsyncImpls = {
__autoLocks.delete(fid);
}catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_LOCK;
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK);
fh.xLock = oldLockType;
}
wTimeEnd();
@ -545,7 +576,7 @@ const vfsAsyncImpls = {
if(undefined===nRead) wTimeEnd();
error("xRead() failed",e,fh);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_READ;
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ);
}
storeAndNotify('xRead',rc);
mTimeEnd();
@ -579,7 +610,7 @@ const vfsAsyncImpls = {
}catch(e){
error("xTruncate():",e,fh);
state.s11n.storeException(2,e);
rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE);
}
wTimeEnd();
storeAndNotify('xTruncate',rc);
@ -619,7 +650,7 @@ const vfsAsyncImpls = {
}catch(e){
error("xWrite():",e,fh);
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR_WRITE;
rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE);
}
wTimeEnd();
storeAndNotify('xWrite',rc);
@ -746,22 +777,16 @@ const waitLoop = async function f(){
/**
waitTime is how long (ms) to wait for each Atomics.wait().
We need to wake up periodically to give the thread a chance
to do other things.
to do other things. If this is too high (e.g. 500ms) then
even two workers/tabs can easily run into locking errors.
*/
const waitTime = 500;
const waitTime = 150;
while(!flagAsyncShutdown){
try {
if('timed-out'===Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, waitTime
)){
if(__autoLocks.size){
/* Release all auto-locks. */
for(const fid of __autoLocks){
const fh = __openFiles[fid];
await closeSyncHandleNoThrow(fh);
log("Auto-unlocked",fid,fh.filenameAbs);
}
}
await closeAutoLocks();
continue;
}
const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
@ -791,7 +816,7 @@ navigator.storage.getDirectory().then(function(d){
const opt = data.args;
state.littleEndian = opt.littleEndian;
state.asyncS11nExceptions = opt.asyncS11nExceptions;
state.verbose = opt.verbose ?? 2;
state.verbose = opt.verbose ?? 1;
state.fileBufferSize = opt.fileBufferSize;
state.sabS11nOffset = opt.sabS11nOffset;
state.sabS11nSize = opt.sabS11nSize;